-
Notifications
You must be signed in to change notification settings - Fork 25
fix(catalog): validate identifier names used to build catalog paths #264
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -18,10 +18,13 @@ | |
|
|
||
| #include "paimon/core/catalog/catalog_utils.h" | ||
|
|
||
| #include <algorithm> | ||
| #include <cctype> | ||
| #include <optional> | ||
|
|
||
| #include "fmt/format.h" | ||
| #include "paimon/catalog/catalog.h" | ||
| #include "paimon/common/utils/string_utils.h" | ||
| #include "paimon/result.h" | ||
|
|
||
| namespace paimon { | ||
|
|
@@ -33,6 +36,27 @@ Status SystemTableError(const Identifier& identifier, const std::string& action) | |
| action, identifier.ToString())); | ||
| } | ||
|
|
||
| /// Rejects names that cannot be used as a single path component: such a name would make the | ||
| /// path built from it escape the directory it is joined to. | ||
| Status CheckValidIdentifierName(const std::string& kind, const std::string& name) { | ||
| const char* reason = nullptr; | ||
| if (StringUtils::IsNullOrWhitespaceOnly(name)) { | ||
| reason = "cannot be empty or whitespace"; | ||
| } else if (name == "." || name == "..") { | ||
| reason = "cannot be '.' or '..'"; | ||
| } else if (name.find('/') != std::string::npos || name.find('\\') != std::string::npos) { | ||
| reason = "cannot contain path separators"; | ||
| } else if (std::any_of(name.begin(), name.end(), [](char c) { | ||
| return std::iscntrl(static_cast<unsigned char>(c)) != 0; | ||
| })) { | ||
| reason = "cannot contain control characters"; | ||
| } | ||
| if (reason != nullptr) { | ||
| return Status::Invalid(fmt::format("{} name {}: '{}'", kind, reason, name)); | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The rejected |
||
| } | ||
| return Status::OK(); | ||
| } | ||
|
|
||
| } // namespace | ||
|
|
||
| bool CatalogUtils::IsSystemDatabase(const std::string& db_name) { | ||
|
|
@@ -70,4 +94,31 @@ Status CatalogUtils::CheckNotBranch(const Identifier& identifier, const std::str | |
| return Status::OK(); | ||
| } | ||
|
|
||
| Status CatalogUtils::CheckValidDatabaseName(const std::string& db_name) { | ||
| return CheckValidIdentifierName("database", db_name); | ||
| } | ||
|
|
||
| Status CatalogUtils::CheckValidTableName(const Identifier& identifier) { | ||
| PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); | ||
| PAIMON_RETURN_NOT_OK(CheckValidIdentifierName("table", data_table_name)); | ||
| PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> branch, identifier.GetBranchName()); | ||
| if (branch) { | ||
| PAIMON_RETURN_NOT_OK(CheckValidIdentifierName("branch", branch.value())); | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This path sends an identifier branch through the generic validator, so a whitespace-only branch is rejected. |
||
| } | ||
| PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> system_table, | ||
| identifier.GetSystemTableName()); | ||
| if (system_table) { | ||
| PAIMON_RETURN_NOT_OK(CheckValidIdentifierName("system table", system_table.value())); | ||
| } | ||
| return Status::OK(); | ||
| } | ||
|
|
||
| Status CatalogUtils::CheckValidBranchName(const std::string& branch) { | ||
| // An empty branch selects the main branch, see BranchManager::NormalizeBranch. | ||
| if (StringUtils::IsNullOrWhitespaceOnly(branch)) { | ||
| return Status::OK(); | ||
| } | ||
| return CheckValidIdentifierName("branch", branch); | ||
| } | ||
|
|
||
| } // namespace paimon | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -87,7 +87,7 @@ Status FileSystemCatalog::CreateDatabaseImpl(const std::string& db_name, | |
| fmt::join(options, ", ")); | ||
| PAIMON_LOG_DEBUG(logger_, "%s", log_msg.c_str()); | ||
| } | ||
| std::string db_path = NewDatabasePath(warehouse_, db_name); | ||
| PAIMON_ASSIGN_OR_RAISE(std::string db_path, NewDatabasePath(warehouse_, db_name)); | ||
| PAIMON_RETURN_NOT_OK(fs_->Mkdirs(db_path)); | ||
| return Status::OK(); | ||
| } | ||
|
|
@@ -96,14 +96,18 @@ Result<bool> FileSystemCatalog::DatabaseExists(const std::string& db_name) const | |
| if (CatalogUtils::IsSystemDatabase(db_name)) { | ||
| return true; | ||
| } | ||
| return fs_->Exists(NewDatabasePath(warehouse_, db_name)); | ||
| PAIMON_ASSIGN_OR_RAISE(std::string db_path, NewDatabasePath(warehouse_, db_name)); | ||
| return fs_->Exists(db_path); | ||
| } | ||
|
|
||
| Result<bool> FileSystemCatalog::TableExists(const Identifier& identifier) const { | ||
| // Handle sys database global tables | ||
| if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { | ||
| return GlobalSystemTableLoader::IsSupported(identifier.GetTableName(), catalog_options_); | ||
| } | ||
| // The branch component is dropped when the data table identifier is rebuilt below, so the | ||
| // identifier is validated as a whole here. | ||
| PAIMON_RETURN_NOT_OK(CatalogUtils::CheckValidTableName(identifier)); | ||
| PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); | ||
| if (is_system_table) { | ||
| PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> system_table_name, | ||
|
|
@@ -123,7 +127,8 @@ Result<bool> FileSystemCatalog::TableExists(const Identifier& identifier) const | |
| } | ||
|
|
||
| std::string FileSystemCatalog::GetDatabaseLocation(const std::string& db_name) const { | ||
| return NewDatabasePath(warehouse_, db_name); | ||
| // An invalid name has no valid location, keep the same convention as RestCatalog. | ||
| return NewDatabasePath(warehouse_, db_name).value_or(""); | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why not return |
||
| } | ||
|
|
||
| Result<std::string> FileSystemCatalog::GetTableLocation(const Identifier& identifier) const { | ||
|
|
@@ -204,16 +209,19 @@ Result<bool> FileSystemCatalog::IsSystemTable(const Identifier& identifier) { | |
| return IsSpecifiedSystemTable(identifier); | ||
| } | ||
|
|
||
| std::string FileSystemCatalog::NewDatabasePath(const std::string& warehouse, | ||
| const std::string& db_name) { | ||
| Result<std::string> FileSystemCatalog::NewDatabasePath(const std::string& warehouse, | ||
| const std::string& db_name) { | ||
| PAIMON_RETURN_NOT_OK(CatalogUtils::CheckValidDatabaseName(db_name)); | ||
| return PathUtil::JoinPath(warehouse, db_name + DB_SUFFIX); | ||
| } | ||
|
|
||
| Result<std::string> FileSystemCatalog::NewDataTablePath(const std::string& warehouse, | ||
| const Identifier& identifier) { | ||
| PAIMON_RETURN_NOT_OK(CatalogUtils::CheckValidTableName(identifier)); | ||
| PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); | ||
| return PathUtil::JoinPath(NewDatabasePath(warehouse, identifier.GetDatabaseName()), | ||
| data_table_name); | ||
| PAIMON_ASSIGN_OR_RAISE(std::string database_path, | ||
| NewDatabasePath(warehouse, identifier.GetDatabaseName())); | ||
| return PathUtil::JoinPath(database_path, data_table_name); | ||
| } | ||
|
|
||
| Result<std::vector<std::string>> FileSystemCatalog::ListDatabases() const { | ||
|
|
@@ -235,7 +243,7 @@ Result<std::vector<std::string>> FileSystemCatalog::ListTables(const std::string | |
| if (CatalogUtils::IsSystemDatabase(db_name)) { | ||
| return GlobalSystemTableLoader::GetSupportedTableNames(catalog_options_); | ||
| } | ||
| std::string database_path = NewDatabasePath(warehouse_, db_name); | ||
| PAIMON_ASSIGN_OR_RAISE(std::string database_path, NewDatabasePath(warehouse_, db_name)); | ||
| std::vector<BasicFileStatus> file_status_list; | ||
| PAIMON_RETURN_NOT_OK(fs_->ListDir(database_path, &file_status_list)); | ||
| std::vector<std::string> table_names; | ||
|
|
@@ -284,6 +292,9 @@ Result<std::shared_ptr<Schema>> FileSystemCatalog::LoadTableSchema( | |
| system_table->ArrowSchema()); | ||
| return std::make_shared<SystemTableSchema>(std::move(arrow_schema)); | ||
| } | ||
| // The branch component is dropped when the data table identifier is rebuilt below, so the | ||
| // identifier is validated as a whole here. | ||
| PAIMON_RETURN_NOT_OK(CatalogUtils::CheckValidTableName(identifier)); | ||
| PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); | ||
| if (is_system_table) { | ||
| PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> system_table_name, | ||
|
|
@@ -342,7 +353,7 @@ Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_n | |
| } | ||
| } | ||
|
|
||
| std::string db_path = NewDatabasePath(warehouse_, name); | ||
| PAIMON_ASSIGN_OR_RAISE(std::string db_path, NewDatabasePath(warehouse_, name)); | ||
|
|
||
| if (cascade) { | ||
| // List all tables in the database and drop them | ||
|
|
@@ -511,6 +522,7 @@ Status FileSystemCatalog::RenameTable(const Identifier& from_table, const Identi | |
|
|
||
| Result<std::vector<SnapshotInfo>> FileSystemCatalog::ListSnapshots( | ||
| const Identifier& identifier, const std::string& branch) const { | ||
| PAIMON_RETURN_NOT_OK(CatalogUtils::CheckValidBranchName(branch)); | ||
| PAIMON_ASSIGN_OR_RAISE(bool exists, TableExists(identifier)); | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Branch validation does not cover all filesystem entry points This validation only protects For example, if a valid |
||
| if (!exists) { | ||
| return Status::NotExist(fmt::format("table {} does not exist", identifier.ToString())); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
StringUtils::IsNullOrWhitespaceOnlyandstd::iscntrlclassify individual UTF-8 bytes under the current C locale, while the referenced Rust implementation uses Unicode-awaretrimandchar::is_control. Under the typical C locale, a name consisting of U+2003 EM SPACE or containing U+0085 is accepted here but rejected by Rust. C++ can therefore create catalog objects that Rust cannot operate on, and the result also depends on process locale.