From c27d91a57302e7e37831ba47bb2e82a8099a7051 Mon Sep 17 00:00:00 2001
From: Chris Punches <phanes@forge.silogroup.lan>
Date: Tue, 25 Mar 2025 23:22:15 -0400
Subject: [PATCH] overhaul of metadata generation and management
---
modules/build/build.cpp | 4 +-
modules/build/include/checksums.hpp | 11 +
modules/build/include/cli_parsers.hpp | 2 +-
modules/build/include/commands.hpp | 4 +-
modules/build/include/metadata.hpp | 61 ++-
modules/build/src/checksums.cpp | 58 +++
modules/build/src/cli_parsers.cpp | 6 +-
modules/build/src/commands.cpp | 358 ++++------------
modules/build/src/metadata.cpp | 581 +++++++++++++++++++++++---
modules/build/src/staging.cpp | 6 -
10 files changed, 739 insertions(+), 352 deletions(-)
diff --git a/modules/build/build.cpp b/modules/build/build.cpp
index 57bd782..4a2992c 100644
--- a/modules/build/build.cpp
+++ b/modules/build/build.cpp
@@ -92,8 +92,8 @@ extern "C" int dpm_module_execute(const char* command, int argc, char** argv) {
case CMD_HELP:
return cmd_help(argc, argv);
- case CMD_MANIFEST:
- return cmd_manifest(argc, argv);
+ case CMD_METADATA:
+ return cmd_metadata(argc, argv);
case CMD_SIGN:
return cmd_sign(argc, argv);
diff --git a/modules/build/include/checksums.hpp b/modules/build/include/checksums.hpp
index a8d4288..b259ad1 100644
--- a/modules/build/include/checksums.hpp
+++ b/modules/build/include/checksums.hpp
@@ -55,3 +55,14 @@ std::string get_available_algorithms();
* @return String containing the hexadecimal representation of the checksum, or empty string on error
*/
std::string generate_file_checksum(const std::filesystem::path& file_path);
+
+/**
+ * @brief Generates a checksum of a string using the configured hashing algorithm
+ *
+ * Uses OpenSSL to calculate a cryptographic hash of a string's contents
+ * based on the algorithm specified in the configuration.
+ *
+ * @param input_string The string to be hashed
+ * @return String containing the hexadecimal representation of the checksum, or empty string on error
+ */
+std::string generate_string_checksum(const std::string& input_string);
\ No newline at end of file
diff --git a/modules/build/include/cli_parsers.hpp b/modules/build/include/cli_parsers.hpp
index 110f300..57477fe 100644
--- a/modules/build/include/cli_parsers.hpp
+++ b/modules/build/include/cli_parsers.hpp
@@ -15,7 +15,7 @@ enum Command {
CMD_UNKNOWN, /**< Unknown or unsupported command */
CMD_HELP, /**< Display help information */
CMD_STAGE, /**< Stage a new DPM package */
- CMD_MANIFEST, /**< Regenerate a stage manifest */
+ CMD_METADATA, /**< Regenerate stage metadata */
CMD_SIGN, /**< Sign a package or stage directory */
CMD_SEAL, /**< Seal a package stage directory */
CMD_UNSEAL, /**< Unseal a package stage directory */
diff --git a/modules/build/include/commands.hpp b/modules/build/include/commands.hpp
index bd71ebb..e00dab7 100644
--- a/modules/build/include/commands.hpp
+++ b/modules/build/include/commands.hpp
@@ -29,7 +29,7 @@ int cmd_stage(int argc, char** argv);
* @param argv Array of arguments
* @return 0 on success, non-zero on failure
*/
-int cmd_manifest(int argc, char** argv);
+int cmd_metadata(int argc, char** argv);
/**
* @brief Handler for the sign command
@@ -84,7 +84,7 @@ int cmd_sign_help(int argc, char** argv);
* @param argv Array of arguments
* @return 0 on success, non-zero on failure
*/
-int cmd_manifest_help(int argc, char** argv);
+int cmd_metadata_help(int argc, char** argv);
/**
* @brief Handler for unknown commands
diff --git a/modules/build/include/metadata.hpp b/modules/build/include/metadata.hpp
index a8c4bc2..d1b31d7 100644
--- a/modules/build/include/metadata.hpp
+++ b/modules/build/include/metadata.hpp
@@ -18,10 +18,25 @@
#include <sys/stat.h>
#include <pwd.h>
#include <grp.h>
+#include <map>
#include <dpmdk/include/CommonModuleAPI.hpp>
#include "checksums.hpp"
+// generates the initial entries for the stage - does not populate data!
+bool metadata_generate_skeleton(const std::filesystem::path& stage_dir);
+
+// sets values in metadata files
+bool metadata_set_simple_value(const std::filesystem::path& stage_dir, const std::string& key, const std::string& value);
+
+// sets initial known values in metadata
+bool metadata_set_initial_known_values(
+ const std::filesystem::path& stage_dir,
+ const std::string& package_name,
+ const std::string& package_version,
+ const std::string& architecture
+);
+
/**
* @brief Updates the contents manifest file for a package stage
*
@@ -32,7 +47,37 @@
* @param package_dir Root directory of the package stage
* @return true if contents manifest generation was successful, false otherwise
*/
-bool generate_contents_manifest(const std::filesystem::path& package_dir);
+bool metadata_generate_contents_manifest_digest(const std::filesystem::path& package_dir);
+
+/**
+ * @brief Refreshes the contents manifest file by updating checksums
+ *
+ * Iterates through the existing CONTENTS_MANIFEST_DIGEST file, rereads each file,
+ * recalculates its checksum, and updates the file with new checksums while
+ * preserving all other fields.
+ *
+ * @param stage_dir Directory path of the package stage
+ * @param force Whether to force the operation even if warnings occur
+ * @return 0 on success, non-zero on failure
+ */
+int metadata_refresh_contents_manifest_digest(const std::string& stage_dir, bool force);
+
+/**
+ * @brief Generates the HOOKS_DIGEST file for a package stage
+ *
+ * Creates the HOOKS_DIGEST file by scanning the hooks directory
+ * and generating a line for each hook file with its checksum and filename.
+ *
+ * @param stage_dir Root directory of the package stage
+ * @return true if hooks digest generation was successful, false otherwise
+ */
+bool metadata_generate_hooks_digest(const std::filesystem::path& stage_dir);
+
+// generates the dynamic entries for the stage
+bool metadata_generate_dynamic_files( const std::filesystem::path& stage_dir );
+
+// refreshes the dynamic entries for the stage
+bool metadata_refresh_dynamic_files( const std::filesystem::path& stage_dir );
/**
* @brief Generates basic metadata files for a package stage
@@ -52,4 +97,16 @@ bool metadata_generate_new(
const std::string& package_name,
const std::string& package_version,
const std::string& architecture
-);
\ No newline at end of file
+);
+
+/**
+ * @brief Generates the PACKAGE_DIGEST for a package stage
+ *
+ * Creates the PACKAGE_DIGEST by generating checksums of CONTENTS_MANIFEST_DIGEST
+ * and HOOKS_DIGEST, concatenating them, and then calculating a checksum of the
+ * concatenation. The result is stored in the PACKAGE_DIGEST file.
+ *
+ * @param stage_dir Root directory of the package stage
+ * @return true if package digest generation was successful, false otherwise
+ */
+bool metadata_generate_package_digest(const std::filesystem::path& stage_dir);
\ No newline at end of file
diff --git a/modules/build/src/checksums.cpp b/modules/build/src/checksums.cpp
index 23a4875..d869187 100644
--- a/modules/build/src/checksums.cpp
+++ b/modules/build/src/checksums.cpp
@@ -169,3 +169,61 @@ std::string generate_file_checksum(const std::filesystem::path& file_path)
return ss.str();
}
+
+std::string generate_string_checksum(const std::string& input_string)
+{
+ // Get configured algorithm
+ std::string algorithm_name = get_configured_hash_algorithm();
+
+ // Initialize OpenSSL
+ OpenSSL_add_all_digests();
+
+ // Get the digest
+ const EVP_MD* md = EVP_get_digestbyname(algorithm_name.c_str());
+ if (!md) {
+ std::string available_algorithms = get_available_algorithms();
+ dpm_log(LOG_FATAL, ("Hash algorithm not supported: " + algorithm_name +
+ ". Available algorithms: " + available_algorithms).c_str());
+ return "";
+ }
+
+ // Initialize OpenSSL EVP context
+ EVP_MD_CTX* mdctx = EVP_MD_CTX_new();
+ if (!mdctx) {
+ dpm_log(LOG_ERROR, "Failed to create OpenSSL EVP context");
+ return "";
+ }
+
+ if (EVP_DigestInit_ex(mdctx, md, nullptr) != 1) {
+ dpm_log(LOG_ERROR, "Failed to initialize digest context");
+ EVP_MD_CTX_free(mdctx);
+ return "";
+ }
+
+ // Update digest with the string contents
+ if (EVP_DigestUpdate(mdctx, input_string.c_str(), input_string.length()) != 1) {
+ dpm_log(LOG_ERROR, "Failed to update digest");
+ EVP_MD_CTX_free(mdctx);
+ return "";
+ }
+
+ // Finalize the digest
+ unsigned char hash[EVP_MAX_MD_SIZE];
+ unsigned int hash_len = 0;
+
+ if (EVP_DigestFinal_ex(mdctx, hash, &hash_len) != 1) {
+ dpm_log(LOG_ERROR, "Failed to finalize digest");
+ EVP_MD_CTX_free(mdctx);
+ return "";
+ }
+
+ EVP_MD_CTX_free(mdctx);
+
+ // Convert binary hash to hexadecimal string
+ std::stringstream ss;
+ for (unsigned int i = 0; i < hash_len; i++) {
+ ss << std::hex << std::setw(2) << std::setfill('0') << static_cast<int>(hash[i]);
+ }
+
+ return ss.str();
+}
\ No newline at end of file
diff --git a/modules/build/src/cli_parsers.cpp b/modules/build/src/cli_parsers.cpp
index 5f5595e..ad87296 100644
--- a/modules/build/src/cli_parsers.cpp
+++ b/modules/build/src/cli_parsers.cpp
@@ -237,9 +237,9 @@ Command parse_command(const char* cmd_str) {
return CMD_STAGE;
}
- // Check for manifest command, including when it has additional arguments
- if (strncmp(cmd_str, "manifest", 8) == 0) {
- return CMD_MANIFEST;
+ // Check for command, including when it has additional arguments
+ if (strncmp(cmd_str, "metadata", 8) == 0) {
+ return CMD_METADATA;
}
// Check for sign command, including when it has additional arguments
diff --git a/modules/build/src/commands.cpp b/modules/build/src/commands.cpp
index 2454515..6e3413b 100644
--- a/modules/build/src/commands.cpp
+++ b/modules/build/src/commands.cpp
@@ -1,261 +1,15 @@
#include "commands.hpp"
-/**
- * @brief Refreshes the contents manifest file by updating checksums
- *
- * Iterates through the existing CONTENTS_MANIFEST_DIGEST file, rereads each file,
- * recalculates its checksum, and updates the file with new checksums while
- * preserving all other fields.
- *
- * @param stage_dir Directory path of the package stage
- * @param force Whether to force the operation even if warnings occur
- * @return 0 on success, non-zero on failure
- */
-static int refresh_contents_manifest(const std::string& stage_dir, bool force) {
- dpm_log(LOG_INFO, ("Refreshing package manifest for: " + stage_dir).c_str());
-
- std::filesystem::path package_dir = std::filesystem::path(stage_dir);
- std::filesystem::path contents_dir = package_dir / "contents";
- std::filesystem::path manifest_path = package_dir / "metadata" / "CONTENTS_MANIFEST_DIGEST";
-
- // Check if contents directory exists
- if (!std::filesystem::exists(contents_dir)) {
- dpm_log(LOG_ERROR, ("Contents directory does not exist: " + contents_dir.string()).c_str());
- return 1;
- }
-
- // Map to track all files in the contents directory
- std::map<std::filesystem::path, bool> all_content_files;
-
- // Populate map with all files in contents directory
- for (const auto& entry : std::filesystem::recursive_directory_iterator(contents_dir)) {
- if (!std::filesystem::is_directory(entry)) {
- // Store path relative to contents directory
- std::filesystem::path relative_path = std::filesystem::relative(entry.path(), contents_dir);
- all_content_files[relative_path] = false; // Not processed yet
- }
- }
-
- // Check if manifest file exists
- bool manifest_exists = std::filesystem::exists(manifest_path);
-
- // Create a temporary file for the updated manifest
- std::filesystem::path temp_manifest_path = manifest_path.string() + ".tmp";
- std::ofstream temp_manifest_file(temp_manifest_path);
- if (!temp_manifest_file.is_open()) {
- dpm_log(LOG_ERROR, ("Failed to create temporary manifest file: " + temp_manifest_path.string()).c_str());
- return 1;
- }
-
- // Log which hash algorithm is being used
- std::string hash_algorithm = get_configured_hash_algorithm();
- dpm_log(LOG_INFO, ("Refreshing contents manifest using " + hash_algorithm + " checksums...").c_str());
-
- int updated_files = 0;
- int new_files = 0;
-
- // First process existing manifest file if it exists
- if (manifest_exists) {
- std::ifstream manifest_file(manifest_path);
- if (!manifest_file.is_open()) {
- dpm_log(LOG_ERROR, ("Failed to open manifest file for reading: " + manifest_path.string()).c_str());
- temp_manifest_file.close();
- std::filesystem::remove(temp_manifest_path);
- return 1;
- }
-
- std::string line;
- int line_number = 0;
-
- // Process each line in the manifest
- while (std::getline(manifest_file, line)) {
- line_number++;
-
- // Skip empty lines
- if (line.empty()) {
- temp_manifest_file << line << std::endl;
- continue;
- }
-
- // Parse the line into its components
- std::istringstream iss(line);
- char control_designation;
- std::string checksum, permissions, ownership, file_path;
-
- // Extract components (C checksum permissions owner:group /path/to/file)
- iss >> control_designation >> checksum >> permissions >> ownership;
-
- // The file path might contain spaces, so we need to get the rest of the line
- std::getline(iss >> std::ws, file_path);
-
- // Skip if we couldn't parse the line correctly
- if (file_path.empty()) {
- dpm_log(LOG_WARN, ("Skipping malformed line " + std::to_string(line_number) + ": " + line).c_str());
- temp_manifest_file << line << std::endl;
- continue;
- }
-
- // Remove leading slash from file_path if present
- if (file_path[0] == '/') {
- file_path = file_path.substr(1);
- }
-
- // Mark this file as processed
- std::filesystem::path relative_path(file_path);
- if (all_content_files.find(relative_path) != all_content_files.end()) {
- all_content_files[relative_path] = true; // Mark as processed
- }
-
- // Construct the full path to the file in the contents directory
- std::filesystem::path full_file_path = contents_dir / file_path;
-
- // Check if the file exists
- if (!std::filesystem::exists(full_file_path)) {
- dpm_log(LOG_WARN, ("File not found in contents directory: " + full_file_path.string()).c_str());
- // Keep the original line
- temp_manifest_file << control_designation << " "
- << checksum << " "
- << permissions << " "
- << ownership << " "
- << "/" << file_path << std::endl;
- continue;
- }
-
- // Calculate new checksum
- std::string new_checksum = generate_file_checksum(full_file_path);
- if (new_checksum.empty()) {
- dpm_log(LOG_ERROR, ("Failed to generate checksum for: " + full_file_path.string()).c_str());
- manifest_file.close();
- temp_manifest_file.close();
- std::filesystem::remove(temp_manifest_path);
- return 1;
- }
-
- // Write updated line to the temporary file
- temp_manifest_file << control_designation << " "
- << new_checksum << " "
- << permissions << " "
- << ownership << " "
- << "/" << file_path << std::endl;
-
- // Count updated files (only if checksum actually changed)
- if (new_checksum != checksum) {
- updated_files++;
- }
- }
-
- manifest_file.close();
- }
-
- // Now process any new files not in the manifest
- for (const auto& [file_path, processed] : all_content_files) {
- // Skip if already processed from manifest
- if (processed) {
- continue;
- }
-
- // This is a new file
- std::filesystem::path full_file_path = contents_dir / file_path;
-
- // Get file stats for permissions
- struct stat file_stat;
- if (stat(full_file_path.c_str(), &file_stat) != 0) {
- dpm_log(LOG_ERROR, ("Failed to get file stats for: " + full_file_path.string()).c_str());
- continue;
- }
-
- // Format permissions as octal
- char perms[5];
- snprintf(perms, sizeof(perms), "%04o", file_stat.st_mode & 07777);
-
- // Get owner and group information
- struct passwd* pw = getpwuid(file_stat.st_uid);
- struct group* gr = getgrgid(file_stat.st_gid);
-
- std::string owner;
- if (pw) {
- owner = pw->pw_name;
- } else {
- owner = std::to_string(file_stat.st_uid);
- }
-
- std::string group;
- if (gr) {
- group = gr->gr_name;
- } else {
- group = std::to_string(file_stat.st_gid);
- }
-
- std::string ownership = owner + ":" + group;
-
- // Calculate checksum
- std::string checksum = generate_file_checksum(full_file_path);
- if (checksum.empty()) {
- dpm_log(LOG_ERROR, ("Failed to generate checksum for: " + full_file_path.string()).c_str());
- continue;
- }
-
- // By default, mark new files as controlled ('C')
- char control_designation = 'C';
-
- // Write new line to the temporary file
- temp_manifest_file << control_designation << " "
- << checksum << " "
- << perms << " "
- << ownership << " "
- << "/" << file_path.string() << std::endl;
-
- new_files++;
- }
-
- temp_manifest_file.close();
-
- // Replace the original file with the temporary file
- try {
- std::filesystem::rename(temp_manifest_path, manifest_path);
- } catch (const std::filesystem::filesystem_error& e) {
- dpm_log(LOG_ERROR, ("Failed to update manifest file: " + std::string(e.what())).c_str());
- std::filesystem::remove(temp_manifest_path);
- return 1;
- }
-
- // Log results
- if (updated_files > 0) {
- dpm_log(LOG_INFO, ("Updated checksums for " + std::to_string(updated_files) + " existing file(s).").c_str());
- }
- if (new_files > 0) {
- dpm_log(LOG_INFO, ("Added " + std::to_string(new_files) + " new file(s) to manifest.").c_str());
- }
-
- return 0;
-}
-
-static int generate_contents_manifest(
- const std::string& package_dir,
- const std::string& package_name,
- const std::string& package_version,
- const std::string& architecture,
- bool force
-) {
- dpm_log(LOG_INFO, ("Generating content manifest for: " + package_dir).c_str());
-
- // Generate the metadata files using the provided information
- if (!metadata_generate_new(std::filesystem::path(package_dir), package_name, package_version, architecture)) {
- dpm_log(LOG_ERROR, "Failed to generate metadata.");
- return 1;
- }
-
- dpm_log(LOG_INFO, "Package content manifest generated successfully.");
- return 0;
-}
-
-int cmd_manifest(int argc, char** argv) {
+int cmd_metadata(int argc, char** argv) {
// Parse command line options
bool force = false;
bool refresh = false;
bool verbose = false;
bool show_help = false;
- std::string package_dir = "";
+ std::string stage_dir = "";
+ std::string package_name = "";
+ std::string package_version = "";
+ std::string architecture = "";
// Process command-line arguments
for (int i = 1; i < argc; i++) {
@@ -269,29 +23,38 @@ int cmd_manifest(int argc, char** argv) {
verbose = true;
} else if (arg == "-h" || arg == "--help" || arg == "help") {
show_help = true;
- } else if ((arg == "-p" || arg == "--package-dir") && i + 1 < argc) {
- package_dir = argv[i + 1];
+ } else if ((arg == "-s" || arg == "--stage") && i + 1 < argc) {
+ stage_dir = argv[i + 1];
+ i++; // Skip the next argument
+ } else if ((arg == "-n" || arg == "--name") && i + 1 < argc) {
+ package_name = argv[i + 1];
+ i++; // Skip the next argument
+ } else if ((arg == "-V" || arg == "--version") && i + 1 < argc) {
+ package_version = argv[i + 1];
+ i++; // Skip the next argument
+ } else if ((arg == "-a" || arg == "--architecture") && i + 1 < argc) {
+ architecture = argv[i + 1];
i++; // Skip the next argument
}
}
// If help was requested, show it and return
if (show_help) {
- return cmd_manifest_help(argc, argv);
+ return cmd_metadata_help(argc, argv);
}
- // Validate that package directory is provided
- if (package_dir.empty()) {
- dpm_log(LOG_ERROR, "Package directory is required (--package-dir/-p)");
- return cmd_manifest_help(argc, argv);
+ // Validate that stage directory is provided
+ if (stage_dir.empty()) {
+ dpm_log(LOG_ERROR, "Package stage directory is required (--stage/-s)");
+ return cmd_metadata_help(argc, argv);
}
// Expand path if needed
- package_dir = expand_path(package_dir);
+ stage_dir = expand_path(stage_dir);
- // Check if package directory exists
- if (!std::filesystem::exists(package_dir)) {
- dpm_log(LOG_ERROR, ("Package directory does not exist: " + package_dir).c_str());
+ // Check if stage directory exists
+ if (!std::filesystem::exists(stage_dir)) {
+ dpm_log(LOG_ERROR, ("Stage directory does not exist: " + stage_dir).c_str());
return 1;
}
@@ -302,20 +65,44 @@ int cmd_manifest(int argc, char** argv) {
// Call the appropriate function based on the refresh flag
if (refresh) {
- int result = refresh_contents_manifest(package_dir, force);
- if (result != 0) {
- dpm_log(LOG_ERROR, "Failed to refresh contents manifest.");
- return result;
- }
- dpm_log(LOG_INFO, "Contents manifest refreshed successfully.");
- return 0;
- } else {
- bool success = generate_contents_manifest(std::filesystem::path(package_dir));
+ // For refresh mode, we only need the stage directory
+ bool success = metadata_refresh_dynamic_files(stage_dir);
if (!success) {
- dpm_log(LOG_ERROR, "Failed to generate contents manifest.");
+ dpm_log(LOG_ERROR, "Failed to refresh metadata files.");
return 1;
}
- dpm_log(LOG_INFO, "Contents manifest generated successfully.");
+ dpm_log(LOG_INFO, "Metadata files refreshed successfully.");
+ return 0;
+ } else {
+ // For generate mode, we need additional parameters
+ if (package_name.empty()) {
+ dpm_log(LOG_ERROR, "Package name is required for metadata generation (--name/-n)");
+ return cmd_metadata_help(argc, argv);
+ }
+
+ if (package_version.empty()) {
+ dpm_log(LOG_ERROR, "Package version is required for metadata generation (--version/-V)");
+ return cmd_metadata_help(argc, argv);
+ }
+
+ if (architecture.empty()) {
+ dpm_log(LOG_ERROR, "Package architecture is required for metadata generation (--architecture/-a)");
+ return cmd_metadata_help(argc, argv);
+ }
+
+ bool success = metadata_generate_new(
+ std::filesystem::path(stage_dir),
+ package_name,
+ package_version,
+ architecture
+ );
+
+ if (!success) {
+ dpm_log(LOG_ERROR, "Failed to generate metadata files.");
+ return 1;
+ }
+
+ dpm_log(LOG_INFO, "Metadata files generated successfully.");
return 0;
}
}
@@ -488,7 +275,7 @@ int cmd_help(int argc, char** argv) {
dpm_log(LOG_INFO, "");
dpm_log(LOG_INFO, "Available commands:");
dpm_log(LOG_INFO, " stage - Stage a new DPM package directory");
- dpm_log(LOG_INFO, " manifest - Generate or refresh package manifest");
+ dpm_log(LOG_INFO, " metadata - Generate or refresh package metadata");
dpm_log(LOG_INFO, " sign - Sign a package or package stage directory");
dpm_log(LOG_INFO, " seal - Seal a package stage directory into final format");
dpm_log(LOG_INFO, " unseal - Unseal a package back to stage format");
@@ -509,16 +296,29 @@ int cmd_unknown(const char* command, int argc, char** argv) {
return 1;
}
-int cmd_manifest_help(int argc, char** argv) {
- dpm_log(LOG_INFO, "Usage: dpm build manifest [options]");
+int cmd_metadata_help(int argc, char** argv) {
+ dpm_log(LOG_INFO, "Usage: dpm build metadata [options]");
dpm_log(LOG_INFO, "");
dpm_log(LOG_INFO, "Options:");
- dpm_log(LOG_INFO, " -p, --package-dir DIR Package directory path (required)");
- dpm_log(LOG_INFO, " -r, --refresh Refresh existing manifest (default: generate new)");
- dpm_log(LOG_INFO, " -f, --force Force manifest operation even if warnings occur");
+ dpm_log(LOG_INFO, " -s, --stage DIR Package stage directory path (required)");
+ dpm_log(LOG_INFO, " -r, --refresh Refresh existing metadata (use for updating)");
+ dpm_log(LOG_INFO, "");
+ dpm_log(LOG_INFO, "For new metadata generation (when not using --refresh):");
+ dpm_log(LOG_INFO, " -n, --name NAME Package name (required for new generation)");
+ dpm_log(LOG_INFO, " -V, --version VERSION Package version (required for new generation)");
+ dpm_log(LOG_INFO, " -a, --architecture ARCH Package architecture (required for new generation)");
+ dpm_log(LOG_INFO, "");
+ dpm_log(LOG_INFO, "Additional options:");
+ dpm_log(LOG_INFO, " -f, --force Force operation even if warnings occur");
dpm_log(LOG_INFO, " -v, --verbose Enable verbose output");
dpm_log(LOG_INFO, " -h, --help Display this help message");
dpm_log(LOG_INFO, "");
+ dpm_log(LOG_INFO, "Examples:");
+ dpm_log(LOG_INFO, " # Refresh metadata in an existing package stage:");
+ dpm_log(LOG_INFO, " dpm build metadata --stage=./my-package-1.0.x86_64 --refresh");
+ dpm_log(LOG_INFO, "");
+ dpm_log(LOG_INFO, " # Generate new metadata for a package stage:");
+ dpm_log(LOG_INFO, " dpm build metadata --stage=./my-package-1.0.x86_64 --name=my-package --version=1.0 --architecture=x86_64");
return 0;
}
diff --git a/modules/build/src/metadata.cpp b/modules/build/src/metadata.cpp
index 7e3dddb..3421510 100644
--- a/modules/build/src/metadata.cpp
+++ b/modules/build/src/metadata.cpp
@@ -12,44 +12,30 @@
#include "metadata.hpp"
-bool metadata_generate_new(
- const std::filesystem::path& package_dir,
- const std::string& package_name,
- const std::string& package_version,
- const std::string& architecture
-) {
+// generates the initial entries for the stage - does not populate data!
+bool metadata_generate_skeleton(const std::filesystem::path& stage_dir) {
+ // generates empty files, such as when generating a new stage
+
+ // determine the path to the metadata directory
+ std::filesystem::path metadata_dir = stage_dir / "metadata";
+
+ // Check if metadata directory exists and is a directory
+ if (!std::filesystem::exists(metadata_dir)) {
+ dpm_log(LOG_ERROR, ("Metadata directory does not exist: " + metadata_dir.string()).c_str());
+ return false;
+ }
+
+ if (!std::filesystem::is_directory(metadata_dir)) {
+ dpm_log(LOG_ERROR, ("Metadata path exists but is not a directory: " + metadata_dir.string()).c_str());
+ return false;
+ }
+
try {
- std::filesystem::path metadata_dir = package_dir / "metadata";
-
- // Create NAME file
- {
- std::ofstream name_file(metadata_dir / "NAME");
- if (name_file.is_open()) {
- name_file << package_name;
- name_file.close();
- }
- }
-
- // Create VERSION file
- {
- std::ofstream version_file(metadata_dir / "VERSION");
- if (version_file.is_open()) {
- version_file << package_version;
- version_file.close();
- }
- }
-
- // Create ARCHITECTURE file
- {
- std::ofstream arch_file(metadata_dir / "ARCHITECTURE");
- if (arch_file.is_open()) {
- arch_file << architecture;
- arch_file.close();
- }
- }
-
- // Create empty placeholder files for other metadata
+ // Create empty placeholder files for all metadata
std::vector<std::string> metadata_files = {
+ "NAME",
+ "VERSION",
+ "ARCHITECTURE",
"AUTHOR",
"MAINTAINER",
"DEPENDENCIES",
@@ -68,33 +54,94 @@ bool metadata_generate_new(
std::ofstream metadata_file(metadata_dir / file_name);
metadata_file.close();
}
-
- dpm_log(LOG_INFO, "Created metadata files");
-
- // Update the contents manifest
- if (!generate_contents_manifest(package_dir)) {
- dpm_log(LOG_ERROR, "Failed to update contents manifest");
- return false;
- }
-
- return true;
} catch (const std::exception& e) {
dpm_log(LOG_ERROR, ("Failed to create metadata files: " + std::string(e.what())).c_str());
return false;
}
+
+ dpm_log(LOG_INFO, "Metadata skeleton generated.");
+ return true;
}
-/**
- * @brief Updates the contents manifest file for a package
- *
- * Creates the CONTENTS_MANIFEST_DIGEST file by scanning the contents directory
- * and generating a line for each file with control designation,
- * checksum, permissions, ownership, and path information.
- *
- * @param package_dir Root directory of the package being staged
- * @return true if manifest generation was successful, false otherwise
- */
-bool generate_contents_manifest(const std::filesystem::path& package_dir)
+bool metadata_set_simple_value(const std::filesystem::path& stage_dir, const std::string& key, const std::string& value)
+{
+ // populates single-line entries
+
+ // Determine the path to the metadata file
+ std::filesystem::path metadata_file_path = stage_dir / "metadata" / key;
+
+ // Check if the metadata directory exists
+ std::filesystem::path metadata_dir = stage_dir / "metadata";
+ if (!std::filesystem::exists(metadata_dir)) {
+ dpm_log(LOG_ERROR, ("Metadata directory does not exist: " + metadata_dir.string()).c_str());
+ return false;
+ }
+
+ if (!std::filesystem::is_directory(metadata_dir)) {
+ dpm_log(LOG_ERROR, ("Metadata path exists but is not a directory: " + metadata_dir.string()).c_str());
+ return false;
+ }
+
+ // Check if the metadata file exists
+ if (!std::filesystem::exists(metadata_file_path)) {
+ dpm_log(LOG_ERROR, ("Metadata file does not exist: " + metadata_file_path.string()).c_str());
+ return false;
+ }
+
+ try {
+ // Open the file for writing (will overwrite existing content)
+ std::ofstream file(metadata_file_path);
+ if (!file.is_open()) {
+ dpm_log(LOG_ERROR, ("Failed to open metadata file for writing: " + metadata_file_path.string()).c_str());
+ return false;
+ }
+
+ // Write the value to the file
+ file << value;
+ file.close();
+
+ dpm_log(LOG_INFO, ("Set metadata " + key + " to: " + value).c_str());
+ return true;
+ } catch (const std::exception& e) {
+ dpm_log(LOG_ERROR, ("Failed to write metadata value: " + std::string(e.what())).c_str());
+ return false;
+ }
+}
+
+bool metadata_set_initial_known_values(
+ const std::filesystem::path& stage_dir,
+ const std::string& package_name,
+ const std::string& package_version,
+ const std::string& architecture
+) {
+ std::filesystem::path metadata_dir = stage_dir / "metadata";
+
+ std::filesystem::path name_file = metadata_dir / "NAME";
+ std::filesystem::path version_file = metadata_dir / "VERSION";
+ std::filesystem::path architecture_file = metadata_dir / "ARCHITECTURE";
+
+ if (!metadata_set_simple_value( stage_dir, "NAME", package_name ))
+ {
+ dpm_log( LOG_FATAL, "Failed to set 'NAME'." );
+ return false;
+ }
+
+ if (!metadata_set_simple_value( stage_dir, "VERSION", package_version ))
+ {
+ dpm_log( LOG_FATAL, "Failed to set 'VERSION'." );
+ return false;
+ }
+
+ if (!metadata_set_simple_value( stage_dir, "ARCHITECTURE", architecture ))
+ {
+ dpm_log( LOG_FATAL, "Failed to set 'ARCHITECTURE'." );
+ return false;
+ }
+
+ return true;
+}
+
+bool metadata_generate_contents_manifest_digest(const std::filesystem::path& package_dir)
{
try {
std::filesystem::path contents_dir = package_dir / "contents";
@@ -182,4 +229,424 @@ bool generate_contents_manifest(const std::filesystem::path& package_dir)
}
}
+int metadata_refresh_contents_manifest_digest(const std::string& stage_dir, bool force) {
+ dpm_log(LOG_INFO, ("Refreshing package manifest for: " + stage_dir).c_str());
+ std::filesystem::path package_dir = std::filesystem::path(stage_dir);
+ std::filesystem::path contents_dir = package_dir / "contents";
+ std::filesystem::path manifest_path = package_dir / "metadata" / "CONTENTS_MANIFEST_DIGEST";
+
+ // Check if contents directory exists
+ if (!std::filesystem::exists(contents_dir)) {
+ dpm_log(LOG_ERROR, ("Contents directory does not exist: " + contents_dir.string()).c_str());
+ return 1;
+ }
+
+ // Map to track all files in the contents directory
+ std::map<std::filesystem::path, bool> all_content_files;
+
+ // Populate map with all files in contents directory
+ for (const auto& entry : std::filesystem::recursive_directory_iterator(contents_dir)) {
+ if (!std::filesystem::is_directory(entry)) {
+ // Store path relative to contents directory
+ std::filesystem::path relative_path = std::filesystem::relative(entry.path(), contents_dir);
+ all_content_files[relative_path] = false; // Not processed yet
+ }
+ }
+
+ // Check if manifest file exists
+ bool manifest_exists = std::filesystem::exists(manifest_path);
+
+ // Create a temporary file for the updated manifest
+ std::filesystem::path temp_manifest_path = manifest_path.string() + ".tmp";
+ std::ofstream temp_manifest_file(temp_manifest_path);
+ if (!temp_manifest_file.is_open()) {
+ dpm_log(LOG_ERROR, ("Failed to create temporary manifest file: " + temp_manifest_path.string()).c_str());
+ return 1;
+ }
+
+ // Log which hash algorithm is being used
+ std::string hash_algorithm = get_configured_hash_algorithm();
+ dpm_log(LOG_INFO, ("Refreshing contents manifest using " + hash_algorithm + " checksums...").c_str());
+
+ int updated_files = 0;
+ int new_files = 0;
+
+ // First process existing manifest file if it exists
+ if (manifest_exists) {
+ std::ifstream manifest_file(manifest_path);
+ if (!manifest_file.is_open()) {
+ dpm_log(LOG_ERROR, ("Failed to open manifest file for reading: " + manifest_path.string()).c_str());
+ temp_manifest_file.close();
+ std::filesystem::remove(temp_manifest_path);
+ return 1;
+ }
+
+ std::string line;
+ int line_number = 0;
+
+ // Process each line in the manifest
+ while (std::getline(manifest_file, line)) {
+ line_number++;
+
+ // Skip empty lines
+ if (line.empty()) {
+ temp_manifest_file << line << std::endl;
+ continue;
+ }
+
+ // Parse the line into its components
+ std::istringstream iss(line);
+ char control_designation;
+ std::string checksum, permissions, ownership, file_path;
+
+ // Extract components (C checksum permissions owner:group /path/to/file)
+ iss >> control_designation >> checksum >> permissions >> ownership;
+
+ // The file path might contain spaces, so we need to get the rest of the line
+ std::getline(iss >> std::ws, file_path);
+
+ // Skip if we couldn't parse the line correctly
+ if (file_path.empty()) {
+ dpm_log(LOG_WARN, ("Skipping malformed line " + std::to_string(line_number) + ": " + line).c_str());
+ temp_manifest_file << line << std::endl;
+ continue;
+ }
+
+ // Remove leading slash from file_path if present
+ if (file_path[0] == '/') {
+ file_path = file_path.substr(1);
+ }
+
+ // Mark this file as processed
+ std::filesystem::path relative_path(file_path);
+ if (all_content_files.find(relative_path) != all_content_files.end()) {
+ all_content_files[relative_path] = true; // Mark as processed
+ }
+
+ // Construct the full path to the file in the contents directory
+ std::filesystem::path full_file_path = contents_dir / file_path;
+
+ // Check if the file exists
+ if (!std::filesystem::exists(full_file_path)) {
+ dpm_log(LOG_WARN, ("File not found in contents directory: " + full_file_path.string()).c_str());
+ // Keep the original line
+ temp_manifest_file << control_designation << " "
+ << checksum << " "
+ << permissions << " "
+ << ownership << " "
+ << "/" << file_path << std::endl;
+ continue;
+ }
+
+ // Calculate new checksum
+ std::string new_checksum = generate_file_checksum(full_file_path);
+ if (new_checksum.empty()) {
+ dpm_log(LOG_ERROR, ("Failed to generate checksum for: " + full_file_path.string()).c_str());
+ manifest_file.close();
+ temp_manifest_file.close();
+ std::filesystem::remove(temp_manifest_path);
+ return 1;
+ }
+
+ // Write updated line to the temporary file
+ temp_manifest_file << control_designation << " "
+ << new_checksum << " "
+ << permissions << " "
+ << ownership << " "
+ << "/" << file_path << std::endl;
+
+ // Count updated files (only if checksum actually changed)
+ if (new_checksum != checksum) {
+ updated_files++;
+ }
+ }
+
+ manifest_file.close();
+ }
+
+ // Now process any new files not in the manifest
+ for (const auto& [file_path, processed] : all_content_files) {
+ // Skip if already processed from manifest
+ if (processed) {
+ continue;
+ }
+
+ // This is a new file
+ std::filesystem::path full_file_path = contents_dir / file_path;
+
+ // Get file stats for permissions
+ struct stat file_stat;
+ if (stat(full_file_path.c_str(), &file_stat) != 0) {
+ dpm_log(LOG_ERROR, ("Failed to get file stats for: " + full_file_path.string()).c_str());
+ continue;
+ }
+
+ // Format permissions as octal
+ char perms[5];
+ snprintf(perms, sizeof(perms), "%04o", file_stat.st_mode & 07777);
+
+ // Get owner and group information
+ struct passwd* pw = getpwuid(file_stat.st_uid);
+ struct group* gr = getgrgid(file_stat.st_gid);
+
+ std::string owner;
+ if (pw) {
+ owner = pw->pw_name;
+ } else {
+ owner = std::to_string(file_stat.st_uid);
+ }
+
+ std::string group;
+ if (gr) {
+ group = gr->gr_name;
+ } else {
+ group = std::to_string(file_stat.st_gid);
+ }
+
+ std::string ownership = owner + ":" + group;
+
+ // Calculate checksum
+ std::string checksum = generate_file_checksum(full_file_path);
+ if (checksum.empty()) {
+ dpm_log(LOG_ERROR, ("Failed to generate checksum for: " + full_file_path.string()).c_str());
+ continue;
+ }
+
+ // By default, mark new files as controlled ('C')
+ char control_designation = 'C';
+
+ // Write new line to the temporary file
+ temp_manifest_file << control_designation << " "
+ << checksum << " "
+ << perms << " "
+ << ownership << " "
+ << "/" << file_path.string() << std::endl;
+
+ new_files++;
+ }
+
+ temp_manifest_file.close();
+
+ // Replace the original file with the temporary file
+ try {
+ std::filesystem::rename(temp_manifest_path, manifest_path);
+ } catch (const std::filesystem::filesystem_error& e) {
+ dpm_log(LOG_ERROR, ("Failed to update manifest file: " + std::string(e.what())).c_str());
+ std::filesystem::remove(temp_manifest_path);
+ return 1;
+ }
+
+ // Log results
+ if (updated_files > 0) {
+ dpm_log(LOG_INFO, ("Updated checksums for " + std::to_string(updated_files) + " existing file(s).").c_str());
+ }
+ if (new_files > 0) {
+ dpm_log(LOG_INFO, ("Added " + std::to_string(new_files) + " new file(s) to manifest.").c_str());
+ }
+
+ return 0;
+}
+
+bool metadata_generate_hooks_digest(const std::filesystem::path& stage_dir)
+{
+ try {
+ std::filesystem::path hooks_dir = stage_dir / "hooks";
+ std::filesystem::path digest_path = stage_dir / "metadata" / "HOOKS_DIGEST";
+
+ // Check if hooks directory exists
+ if (!std::filesystem::exists(hooks_dir)) {
+ dpm_log(LOG_ERROR, ("Hooks directory does not exist: " + hooks_dir.string()).c_str());
+ return false;
+ }
+
+ // Log which hash algorithm is being used
+ std::string hash_algorithm = get_configured_hash_algorithm();
+ dpm_log(LOG_INFO, ("Generating hooks digest using " + hash_algorithm + " checksums...").c_str());
+
+ // Open digest file for writing
+ std::ofstream digest_file(digest_path);
+ if (!digest_file.is_open()) {
+ dpm_log(LOG_ERROR, ("Failed to open hooks digest file for writing: " + digest_path.string()).c_str());
+ return false;
+ }
+
+ // Process each file in the hooks directory
+ for (const auto& entry : std::filesystem::directory_iterator(hooks_dir)) {
+ // Skip directories, we only need to record files
+ if (std::filesystem::is_directory(entry)) {
+ continue;
+ }
+
+ // Get file information
+ std::filesystem::path file_path = entry.path();
+ std::string filename = entry.path().filename().string();
+
+ // Calculate file checksum using the configured algorithm
+ std::string checksum = generate_file_checksum(file_path);
+ if (checksum.empty()) {
+ dpm_log(LOG_FATAL, ("Failed to generate checksum for: " + file_path.string()).c_str());
+ return false;
+ }
+
+ // Write the digest entry
+ // Format: checksum filename
+ digest_file << checksum << " " << filename << "\n";
+ }
+
+ digest_file.close();
+ dpm_log(LOG_INFO, "Hooks digest generated successfully");
+ return true;
+ }
+ catch (const std::exception& e) {
+ dpm_log(LOG_ERROR, ("Failed to generate hooks digest: " + std::string(e.what())).c_str());
+ return false;
+ }
+}
+
+bool metadata_generate_package_digest(const std::filesystem::path& stage_dir)
+{
+ try {
+ std::filesystem::path contents_manifest_path = stage_dir / "metadata" / "CONTENTS_MANIFEST_DIGEST";
+ std::filesystem::path hooks_digest_path = stage_dir / "metadata" / "HOOKS_DIGEST";
+
+ // Check if required files exist
+ if (!std::filesystem::exists(contents_manifest_path)) {
+ dpm_log(LOG_ERROR, ("CONTENTS_MANIFEST_DIGEST not found: " + contents_manifest_path.string()).c_str());
+ return false;
+ }
+
+ if (!std::filesystem::exists(hooks_digest_path)) {
+ dpm_log(LOG_ERROR, ("HOOKS_DIGEST not found: " + hooks_digest_path.string()).c_str());
+ return false;
+ }
+
+ // Log which hash algorithm is being used
+ std::string hash_algorithm = get_configured_hash_algorithm();
+ dpm_log(LOG_INFO, ("Generating package digest using " + hash_algorithm + " checksums...").c_str());
+
+ // Calculate checksums of both files
+ std::string contents_manifest_checksum = generate_file_checksum(contents_manifest_path);
+ if (contents_manifest_checksum.empty()) {
+ dpm_log(LOG_ERROR, ("Failed to generate checksum for: " + contents_manifest_path.string()).c_str());
+ return false;
+ }
+
+ std::string hooks_digest_checksum = generate_file_checksum(hooks_digest_path);
+ if (hooks_digest_checksum.empty()) {
+ dpm_log(LOG_ERROR, ("Failed to generate checksum for: " + hooks_digest_path.string()).c_str());
+ return false;
+ }
+
+ // Concatenate the two checksums
+ std::string combined_checksums = contents_manifest_checksum + hooks_digest_checksum;
+
+ // Calculate checksum of the combined string
+ std::string package_digest = generate_string_checksum(combined_checksums);
+ if (package_digest.empty()) {
+ dpm_log(LOG_ERROR, "Failed to generate checksum of combined checksums");
+ return false;
+ }
+
+ // Set the package digest in the metadata
+ if (!metadata_set_simple_value(stage_dir, "PACKAGE_DIGEST", package_digest)) {
+ dpm_log(LOG_ERROR, "Failed to set PACKAGE_DIGEST value");
+ return false;
+ }
+
+ dpm_log(LOG_INFO, "Package digest generated successfully");
+ return true;
+ }
+ catch (const std::exception& e) {
+ dpm_log(LOG_ERROR, ("Failed to generate package digest: " + std::string(e.what())).c_str());
+ return false;
+ }
+}
+
+// generates the dynamic entries for the stage
+bool metadata_generate_dynamic_files(const std::filesystem::path& stage_dir)
+{
+ // Generate contents manifest
+ dpm_log(LOG_INFO, "Generating contents manifest digest...");
+ if (!metadata_generate_contents_manifest_digest(stage_dir)) {
+ dpm_log(LOG_ERROR, "Failed to generate contents manifest digest");
+ return false;
+ }
+
+ // Generate hooks digest
+ dpm_log(LOG_INFO, "Generating hooks digest...");
+ if (!metadata_generate_hooks_digest(stage_dir)) {
+ dpm_log(LOG_ERROR, "Failed to generate hooks digest");
+ return false;
+ }
+
+ // Generate package digest
+ dpm_log(LOG_INFO, "Generating package digest...");
+ if (!metadata_generate_package_digest(stage_dir)) {
+ dpm_log(LOG_ERROR, "Failed to generate package digest");
+ return false;
+ }
+
+ dpm_log(LOG_INFO, "Dynamic metadata generation completed successfully");
+ return true;
+}
+
+// refreshes the dynamic entries for the stage
+bool metadata_refresh_dynamic_files(const std::filesystem::path& stage_dir)
+{
+ // Refresh contents manifest
+ dpm_log(LOG_INFO, "Refreshing contents manifest digest...");
+ if (metadata_refresh_contents_manifest_digest(stage_dir, false) != 0) {
+ dpm_log(LOG_ERROR, "Failed to refresh contents manifest digest");
+ return false;
+ }
+
+ // Generate hooks digest
+ dpm_log(LOG_INFO, "Regenerating hooks digest...");
+ if (!metadata_generate_hooks_digest(stage_dir)) {
+ dpm_log(LOG_ERROR, "Failed to regenerate hooks digest");
+ return false;
+ }
+
+ // Generate package digest
+ dpm_log(LOG_INFO, "Regenerating package digest...");
+ if (!metadata_generate_package_digest(stage_dir)) {
+ dpm_log(LOG_ERROR, "Failed to regenerate package digest");
+ return false;
+ }
+
+ dpm_log(LOG_INFO, "Dynamic metadata refresh completed successfully");
+ return true;
+}
+
+bool metadata_generate_new(
+ const std::filesystem::path& stage_dir,
+ const std::string& package_name,
+ const std::string& package_version,
+ const std::string& architecture
+)
+{
+ // Step 1: Generate metadata skeleton
+ dpm_log(LOG_INFO, "Generating metadata skeleton...");
+ if (!metadata_generate_skeleton(stage_dir)) {
+ dpm_log(LOG_ERROR, "Failed to generate metadata skeleton");
+ return false;
+ }
+
+ // Step 2: Set initial known values
+ dpm_log(LOG_INFO, "Setting initial metadata values...");
+ if (!metadata_set_initial_known_values(stage_dir, package_name, package_version, architecture)) {
+ dpm_log(LOG_ERROR, "Failed to set initial metadata values");
+ return false;
+ }
+
+ // Step 3: Generate dynamic files
+ dpm_log(LOG_INFO, "Generating dynamic metadata files...");
+ if (!metadata_generate_dynamic_files(stage_dir)) {
+ dpm_log(LOG_ERROR, "Failed to generate dynamic metadata files");
+ return false;
+ }
+
+ dpm_log(LOG_INFO, "Metadata generation completed successfully");
+ return true;
+}
\ No newline at end of file
diff --git a/modules/build/src/staging.cpp b/modules/build/src/staging.cpp
index 54c9ff3..e45b4fe 100644
--- a/modules/build/src/staging.cpp
+++ b/modules/build/src/staging.cpp
@@ -306,12 +306,6 @@ int build_package_stage(
return 1;
}
- // Update the contents manifest
- if (!generate_contents_manifest(package_dir))
- {
- return 1;
- }
-
dpm_log(LOG_INFO, "Package staging completed successfully");
dpm_log(LOG_INFO, ("Package staged at: " + package_dir.string()).c_str());
dpm_log(LOG_INFO, "Next steps:");