interpolate issue resolved

.gitignore

@@ -6,4 +6,5 @@ cmake-build-debug
 cmake-build-release
 CMakeFiles
 CMakeLists.txt
-Makefile
+Makefile
+sample/logs/*

CMakeLists.txt

@@ -3,4 +3,4 @@ project(rex)
 
 set(CMAKE_CXX_STANDARD 14)
 
-add_executable(rex Rex.cpp src/json_support/jsoncpp/json.h src/json_support/jsoncpp/json-forwards.h src/json_support/jsoncpp/jsoncpp.cpp src/logger/Logger.cpp src/logger/Logger.h src/json_support/JSON.cpp src/json_support/JSON.h src/misc/helpers.cpp src/misc/helpers.h src/config/Config.cpp src/config/Config.h src/suite/Suite.cpp src/suite/Suite.h src/suite/Unit.cpp src/suite/Unit.h src/shells/shells.cpp src/shells/shells.h src/plan/Plan.cpp src/plan/Plan.h src/plan/Task.cpp src/plan/Task.h src/lcpex/liblcpex.h src/lcpex/liblcpex.cpp src/lcpex/vpty/libclpex_tty.h src/lcpex/vpty/libclpex_tty.cpp src/lcpex/Contexts.h src/lcpex/Contexts.cpp src/lcpex/helpers.h src/lcpex/string_expansion/string_expansion.h src/lcpex/string_expansion/string_expansion.cpp src/lcpex/vpty/pty_fork_mod/pty_fork.h src/lcpex/vpty/pty_fork_mod/pty_fork.cpp src/lcpex/vpty/pty_fork_mod/pty_master_open.h src/lcpex/vpty/pty_fork_mod/pty_master_open.cpp src/lcpex/vpty/pty_fork_mod/tty_functions.h src/lcpex/vpty/pty_fork_mod/tty_functions.cpp )
+add_executable(rex Rex.cpp src/json_support/jsoncpp/json.h src/json_support/jsoncpp/json-forwards.h src/json_support/jsoncpp/jsoncpp.cpp src/logger/Logger.cpp src/logger/Logger.h src/json_support/JSON.cpp src/json_support/JSON.h src/misc/helpers.cpp src/misc/helpers.h src/config/Config.cpp src/config/Config.h src/suite/Suite.cpp src/suite/Suite.h src/suite/Unit.cpp src/suite/Unit.h src/shells/shells.cpp src/shells/shells.h src/plan/Plan.cpp src/plan/Plan.h src/plan/Task.cpp src/plan/Task.h src/lcpex/helpers.h src/lcpex/helpers.cpp src/lcpex/liblcpex.h src/lcpex/liblcpex.cpp src/lcpex/vpty/libclpex_tty.h src/lcpex/vpty/libclpex_tty.cpp src/lcpex/Contexts.h src/lcpex/Contexts.cpp src/lcpex/helpers.h src/lcpex/string_expansion/string_expansion.h src/lcpex/string_expansion/string_expansion.cpp src/lcpex/vpty/pty_fork_mod/pty_fork.h src/lcpex/vpty/pty_fork_mod/pty_fork.cpp src/lcpex/vpty/pty_fork_mod/pty_master_open.h src/lcpex/vpty/pty_fork_mod/pty_master_open.cpp src/lcpex/vpty/pty_fork_mod/tty_functions.h src/lcpex/vpty/pty_fork_mod/tty_functions.cpp )

Rex.cpp

@@ -15,12 +15,15 @@
 */
 
 #include <iostream>
+#include <string>
+#include <cstring>
 #include <unistd.h>
 #include <getopt.h>
 #include "src/logger/Logger.h"
 #include "src/config/Config.h"
 #include "src/suite/Suite.h"
 #include "src/plan/Plan.h"
+#include "src/misc/helpers.h"
 
 void version_info()
 {
@@ -162,6 +165,8 @@ int main( int argc, char * argv[] )
     interpolate( config_path );
     interpolate( plan_path );
 
+    plan_path = get_absolute_path( plan_path );
+
     // default logging level
     int L_LEVEL = E_INFO;
 

sample/components/independent_test_1.bash

@@ -1,7 +1,7 @@
 #!/usr/bin/bash
 #
 echo test from script
-/usr/bin/dialog --title "This should be one argument" --inputbox "Enter your name:" 0 0
+#/usr/bin/dialog --title "This should be one argument" --inputbox "Enter your name:" 0 0
 
 
 

sample/environments/rex.variables

@@ -1,9 +1,5 @@
 #!/usr/bin/bash
 
-
-
-
-
 set -a
 
 echo "variables file says hello and set a variable named TEST_VAR"

sample/plans/test.plan

@@ -1,5 +1,6 @@
 {
   "plan": [
-    { "name": "independent test 1", "dependencies":  [ null ] }
+    { "name": "independent test 1", "dependencies":  [ null ] },
+    { "name": "independent test 2", "dependencies":  [ null ] }
   ]
 }

sample/rex.config

@@ -1,6 +1,6 @@
 {
   "config": {
-	"project_root": 	"/tmp/sample",
+	"project_root": 	"$HOME/development/internal/rex/sample",
   	"units_path":		"units/",
   	"logs_path": 		"logs/",
 	"shells_path":		"shells/shells.definitions",

sample/units/new.units

@@ -13,8 +13,26 @@
         "active":       		true,
         "required":     		true,
 	"set_user_context": 		true,
-        "user":         		"bagira",
-        "group":        		"bagira",
+        "user":         		"$USER",
+        "group":        		"$USER",
+	"supply_environment":		true,
+        "environment":  		"environments/rex.variables"
+    },
+    {
+        "name":         		"independent test 2",
+        "target":       		"components/independent_test_1.bash --ls -s --arg",
+	"is_shell_command":		true,
+	"shell_definition":		"bash",
+	"force_pty":			true,
+	"set_working_directory":	false,
+	"working_directory":		"",
+        "rectify":      		false,
+        "rectifier":    		"echo rectifier executed",
+        "active":       		true,
+        "required":     		true,
+	"set_user_context": 		true,
+        "user":         		"$USER",
+        "group":        		"$USER",
 	"supply_environment":		true,
         "environment":  		"environments/rex.variables"
     }

src/config/Config.cpp

@@ -72,7 +72,7 @@ void Conf::set_object_s(std::string keyname, std::string & object_member, std::s
     if ( this->get_string(jval_s, keyname) != 0) {
         throw ConfigLoadException( "'" + keyname + "' string is not set in the config file supplied: " + filename );
     } else {
-        interpolate(jval_s);
+        //interpolate(jval_s);
         object_member = jval_s;
     }
     this->slog.log_task( E_DEBUG, "SET_PROPERTY", "'" + keyname + "': " + object_member );
@@ -101,7 +101,7 @@ void Conf::set_object_s_derivedpath(std::string keyname, std::string & object_me
     if ( this->get_string(jval_s, keyname) != 0) {
         throw ConfigLoadException( "'" + keyname + "' string is not set in the config file supplied: " + filename );
     } else {
-        interpolate(jval_s);
+        //interpolate(jval_s);
         object_member = prepend_project_root( jval_s );
     }
     this->slog.log_task( E_DEBUG, "SET_PROPERTY", "'" + keyname + "': " + object_member );
@@ -157,29 +157,7 @@ std::string Conf::prepend_project_root( std::string relative_path)
     return this->project_root + "/" + relative_path;
 }
 
-/**
- * @brief Get the absolute path from a relative path
- *
- * This function takes a relative path and returns the corresponding absolute path.
- * The absolute path is obtained by calling the `realpath` function.
- * If the `realpath` function returns a null pointer, an error message is printed to the standard error stream and an empty string is returned.
- *
- * @param relative_path The relative path to be converted to an absolute path
- *
- * @return The absolute path corresponding to the relative path
- */
-std::string get_absolute_path(const std::string &relative_path)
-{
-    char resolved_path[1024];
-    memset(resolved_path, 0, sizeof(resolved_path));
 
-    if( realpath( relative_path.c_str(), resolved_path) == nullptr ) {
-        std::cerr << "Error resolving path: " << relative_path << std::endl;
-        return "";
-    }
-
-    return std::string(resolved_path);
-}
 
 /**
  * @brief Check if a path exists
@@ -265,8 +243,6 @@ Shell Conf::get_shell_by_name( std::string name ) {
     throw ConfigLoadException("The shell specified ('" + name + "') is not defined in the shell definitions file.");
 }
 
-
-
 /**
  * @class Conf
  * @brief Loads the configuration for the application
@@ -284,9 +260,9 @@ Shell Conf::get_shell_by_name( std::string name ) {
 Conf::Conf(std::string filename, int LOG_LEVEL ): JSON_Loader(LOG_LEVEL ), slog(LOG_LEVEL, "_conf_" )
 {
     this->LOG_LEVEL = LOG_LEVEL;
-    this->slog.log_task( E_DEBUG, "LOAD", "Loading configuration file: " + filename );
 
     interpolate( filename );
+    this->slog.log_task( E_DEBUG, "LOAD", "Loading configuration file: " + filename );
 
     try {
         // load the test file.
@@ -295,6 +271,7 @@ Conf::Conf(std::string filename, int LOG_LEVEL ): JSON_Loader(LOG_LEVEL ), slog(
         this->slog.log( E_FATAL, "Unable to load configuration file: '" + filename + "'. Error: " + e.what());
         throw ConfigLoadException("Parsing error in configuration file.");
     }
+
     Json::Value jbuff;
 
     if ( this->get_serialized( jbuff, "config" ) != 0) {
@@ -305,22 +282,27 @@ Conf::Conf(std::string filename, int LOG_LEVEL ): JSON_Loader(LOG_LEVEL ), slog(
         this->json_root = jbuff;
     }
 
-    set_object_s(               "project_root",     this->project_root,           filename );
+    set_object_s( "project_root",     this->project_root,           filename );
+    interpolate( this->project_root );
+
     // convert to an absolute path after all the interpolation is done.
     this->project_root = get_absolute_path( this->project_root );
-    set_object_s(   "logs_path",        this->logs_path,              filename );
-    this->logs_path = get_absolute_path( this->logs_path );
 
+    set_object_s( "logs_path",        this->logs_path,              filename );
+    interpolate( this->logs_path );
 
     // all other paths are relative to project_root
-    set_object_s_derivedpath(   "units_path",       this->units_path,             filename );
-    set_object_s_derivedpath(   "shells_path",      this->shell_definitions_path, filename );
+    set_object_s_derivedpath( "units_path",       this->units_path,             filename );
+    interpolate( this->units_path );
+
+    set_object_s_derivedpath( "shells_path",      this->shell_definitions_path, filename );
+    interpolate( this->shell_definitions_path );
 
     // ensure these paths exists, with exception to the logs_path, which will be created at runtime
     this->slog.log_task( E_DEBUG, "SANITY_CHECKS", "Checking for sanity..." );
-    checkPathExists(            "project_root",     this->project_root );
-    checkPathExists(            "units_path",       this->units_path );
-    checkPathExists(            "shells_path",      this->shell_definitions_path );
+    checkPathExists( "project_root",     this->project_root );
+    checkPathExists( "units_path",       this->units_path );
+    checkPathExists( "shells_path",      this->shell_definitions_path );
 
     // shells are scoped beyond plan so they need to be considered part of config
     load_shells();

src/lcpex/helpers.cpp

@@ -0,0 +1,15 @@
+#include "helpers.h"
+
+ssize_t write_all(int fd, const void *buf, size_t count) {
+    const char *p = (const char *)buf;
+    while (count > 0) {
+        ssize_t written = write(fd, p, count);
+        if (written == -1) {
+            if (errno == EINTR || errno == EAGAIN) continue; // Retry
+            return -1; // Other errors
+        }
+        count -= written;
+        p += written;
+    }
+    return 0;
+}

src/lcpex/helpers.h

@@ -1,6 +1,9 @@
 #ifndef LCPEX_HELPERS_H
 #define LCPEX_HELPERS_H
 
+#include <unistd.h>
+#include "errno.h"
+
 // helper for sanity
 enum PIPE_ENDS {
     READ_END = 0,
@@ -16,6 +19,6 @@ enum CHILD_PIPE_NAMES {
 
 #define BUFFER_SIZE 1024
 
-
+ssize_t write_all(int fd, const void *buf, size_t count);
 
 #endif //LCPEX_HELPERS_H

src/lcpex/liblcpex.cpp

@@ -1,5 +1,6 @@
 #include "liblcpex.h"
 
+
 std::string prefix_generator(
         std::string command,
         bool is_shell_command,
@@ -180,7 +181,8 @@ int execute(
     // before we fork the process, so that the child process will inherit the environment
     // from the parent process
     if ( environment_supplied ) {
-        clearenv();
+        // this breaks reuse of env variables in between executions
+        //clearenv();
     }
 
     // create the pipes for the child process to write and read from using its stdin/stdout/stderr
@@ -265,6 +267,12 @@ int execute(
             // loop until we've read all the data from the child process
             while ( ! break_out ) {
                 num_files_readable = poll(watched_fds, sizeof(watched_fds) / sizeof(watched_fds[0]), -1);
+                // after the poll() call, add a check to see if both pipes are closed
+                if (!(watched_fds[CHILD_PIPE_NAMES::STDOUT_READ].events & POLLIN) &&
+                    !(watched_fds[CHILD_PIPE_NAMES::STDERR_READ].events & POLLIN)) {
+                    break_out = true;
+                }
+
                 if (num_files_readable == -1) {
                     // error occurred in poll()
                     perror("poll");
@@ -281,9 +289,12 @@ int execute(
                         byte_count = read(watched_fds[this_fd].fd, buf, BUFFER_SIZE);
 
                         if (byte_count == -1) {
-                            // error reading from pipe
-                            perror("read");
-                            exit(EXIT_FAILURE);
+                            if (errno == EAGAIN) { continue; } else {
+                                // error reading from pipe
+                                perror("read");
+                                exit(EXIT_FAILURE);
+                            }
+
                         } else if (byte_count == 0) {
                             // reached EOF on one of the streams but not a HUP
                             // we've read all we can this cycle, so go to the next fd in the for loop
@@ -293,12 +304,12 @@ int execute(
                             // write to stdout,stderr
                             if (this_fd == CHILD_PIPE_NAMES::STDOUT_READ) {
                                 // the child's stdout pipe is readable
-                                write(stdout_log_fh->_fileno, buf, byte_count);
-                                write(STDOUT_FILENO, buf, byte_count);
+                                write_all(stdout_log_fh->_fileno, buf, byte_count);
+                                write_all(STDOUT_FILENO, buf, byte_count);
                             } else if (this_fd == CHILD_PIPE_NAMES::STDERR_READ) {
                                 // the child's stderr pipe is readable
-                                write(stderr_log_fh->_fileno, buf, byte_count);
-                                write(STDERR_FILENO, buf, byte_count);
+                                write_all(stderr_log_fh->_fileno, buf, byte_count);
+                                write_all(STDERR_FILENO, buf, byte_count);
                             } else {
                                 // this should never happen
                                 perror("Logic error!");
@@ -312,15 +323,24 @@ int execute(
                     }
                     if (watched_fds[this_fd].revents & POLLHUP) {
                         // this pipe has hung up
-                        close(watched_fds[this_fd].fd);
-                        break_out = true;
+                        // don't close the file descriptor yet, there might still be data to read
+                        // instead, remove the POLLIN event to avoid getting a POLLHUP event in the next poll() call
+                        watched_fds[this_fd].events &= ~POLLIN;
                     }
-
                 }
             }
             // wait for child to exit, capture status
             waitpid(pid, &status, 0);
 
+            // Drain the pipes before exiting
+            while ((byte_count = read(fd_child_stdout_pipe[READ_END], buf, BUFFER_SIZE)) > 0) {
+                write_all(stdout_log_fh->_fileno, buf, byte_count);
+                write_all(STDOUT_FILENO, buf, byte_count);
+            }
+            while ((byte_count = read(fd_child_stderr_pipe[READ_END], buf, BUFFER_SIZE)) > 0) {
+                write_all(stderr_log_fh->_fileno, buf, byte_count);
+                write_all(STDERR_FILENO, buf, byte_count);
+            }
 
             if WIFEXITED(status) {
                 return WEXITSTATUS(status);

src/lcpex/vpty/libclpex_tty.cpp

@@ -108,7 +108,7 @@ int exec_pty(
     // before we fork the process, so that the child process will inherit the environment
     // from the parent process
     if ( environment_supplied ) {
-        clearenv();
+        //clearenv();
     }
 
     // turn our command string into something execvp can consume
@@ -211,6 +211,11 @@ int exec_pty(
             // loop until we've read all the data from the child process
             while ( ! break_out ) {
                 num_files_readable = poll(watched_fds, sizeof(watched_fds) / sizeof(watched_fds[0]), -1);
+                // after the poll() call, add a check to see if both pipes are closed
+                if (!(watched_fds[1].events & POLLIN) &&
+                    !(watched_fds[2].events & POLLIN)) {
+                    break_out = true;
+                }
 
                 if (num_files_readable == -1) {
                     // error occurred in poll()
@@ -230,9 +235,11 @@ int exec_pty(
                         byte_count = read(watched_fds[this_fd].fd, buf, BUFFER_SIZE);
 
                         if (byte_count == -1) {
-                            // error reading from pipe
-                            safe_perror("read", &ttyOrig );
-                            exit(EXIT_FAILURE);
+                            if (errno == EAGAIN) { continue; } else {
+                                // error reading from pipe
+                                safe_perror("read", &ttyOrig );
+                                exit(EXIT_FAILURE);
+                            }
                         } else if (byte_count == 0) {
                             // reached EOF on one of the streams but not a HUP
                             // we've read all we can this cycle, so go to the next fd in the for loop
@@ -240,17 +247,18 @@ int exec_pty(
                         } else {
                             // byte count was sane
                             // write to stdout,stderr
+
                             if (this_fd == 0) {
                                 // parent stdin received, write to child pty (stdin)
-                                write(masterFd, buf, byte_count);
+                                write_all(masterFd, buf, byte_count);
                             } else if (this_fd == 1 ) {
                                 // child pty sent some stuff, write to parent stdout and log
-                                write(stdout_log_fh->_fileno, buf, byte_count);
-                                write(STDOUT_FILENO, buf, byte_count);
+                                write_all(stdout_log_fh->_fileno, buf, byte_count);
+                                write_all(STDOUT_FILENO, buf, byte_count);
                             } else if ( this_fd == 2 ){
                                 //the child's stderr pipe sent some stuff, write to parent stderr and log
-                                write(stderr_log_fh->_fileno, buf, byte_count);
-                                write(STDERR_FILENO, buf, byte_count);
+                                write_all(stderr_log_fh->_fileno, buf, byte_count);
+                                write_all(STDERR_FILENO, buf, byte_count);
                             } else {
                                 // this should never happen
                                 perror("Logic error!");
@@ -259,21 +267,37 @@ int exec_pty(
                         }
                     }
                     if (watched_fds[this_fd].revents & POLLERR) {
-                        close(watched_fds[this_fd].fd);
-                        //break_out = true;
-                        continue;
+                        //close(watched_fds[this_fd].fd);
+                        break_out = true;
+                        //continue;
                     }
+//                    if (watched_fds[this_fd].revents & POLLHUP) {
+//                        // this pipe has hung up
+//                        close(watched_fds[this_fd].fd);
+//                        break_out = true;
+//                        break;
+//                    }
                     if (watched_fds[this_fd].revents & POLLHUP) {
                         // this pipe has hung up
-                        close(watched_fds[this_fd].fd);
-                        break_out = true;
-                        break;
+                        // don't close the file descriptor yet, there might still be data to read
+                        // instead, remove the POLLIN event to avoid getting a POLLHUP event in the next poll() call
+                        watched_fds[this_fd].events &= ~POLLIN;
                     }
                 }
             }
             // wait for child to exit, capture status
             waitpid(pid, &status, 0);
 
+            while ((byte_count = read(watched_fds[1].fd, buf, BUFFER_SIZE)) > 0) {
+                write_all(stdout_log_fh->_fileno, buf, byte_count);
+                write_all(STDOUT_FILENO, buf, byte_count);
+            }
+
+            while ((byte_count = read(fd_child_stderr_pipe[READ_END], buf, BUFFER_SIZE)) > 0) {
+                write_all(stderr_log_fh->_fileno, buf, byte_count);
+                write_all(STDERR_FILENO, buf, byte_count);
+            }
+
             ttyResetExit( &ttyOrig);
             if WIFEXITED(status) {
                 return WEXITSTATUS(status);
@@ -282,5 +306,4 @@ int exec_pty(
             }
         }
     }
-}
-
+}

src/misc/helpers.cpp

@@ -96,19 +96,57 @@ std::string get_8601()
  * @brief Interpolates the environment variables in the input text
  *
  * This function takes a string reference as input and replaces all occurrences of
- * environment variables in the format `${VAR_NAME}` with their corresponding values.
+ * environment variables in the format `${VAR_NAME}` or `$VAR_NAME` with their corresponding values.
  * If an environment variable is not set, it is replaced with an empty string.
  *
  * @param text The input text to be processed
  */
+//void interpolate( std::string & text )
+//{
+//    static std::regex env1( "\\$\\{([^}]+)\\}" );
+//    static std::regex env2( "\\$([^/]+)" ); // matches $VAR_NAME until a / is found
+//    std::smatch match;
+//    while ( std::regex_search( text, match, env1 ) || std::regex_search( text, match, env2 ) )
+//    {
+//        const char * s = getenv( match[1].str().c_str() );
+//        const std::string var( s == NULL ? "" : s );
+//        text.replace( match[0].first, match[0].second, var );
+//    }
+//}
 void interpolate( std::string & text )
 {
-    static std::regex env( "\\$\\{([^}]+)\\}" );
+    std::regex env1( "\\$\\{([^}]+)\\}" );
+    std::regex env2( "\\$([^/]+)" ); // matches $VAR_NAME until a / is found
     std::smatch match;
-    while ( std::regex_search( text, match, env ) )
+    while ( std::regex_search( text, match, env1 ) || std::regex_search( text, match, env2 ) )
     {
         const char * s = getenv( match[1].str().c_str() );
         const std::string var( s == NULL ? "" : s );
         text.replace( match[0].first, match[0].second, var );
     }
 }
+
+/**
+ * @brief Get the absolute path from a relative path
+ *
+ * This function takes a relative path and returns the corresponding absolute path.
+ * The absolute path is obtained by calling the `realpath` function.
+ * If the `realpath` function returns a null pointer, an error message is printed to the standard error stream and an empty string is returned.
+ *
+ * @param relative_path The relative path to be converted to an absolute path
+ *
+ * @return The absolute path corresponding to the relative path
+ */
+std::string get_absolute_path(const std::string &relative_path)
+{
+    char resolved_path[1024];
+    memset(resolved_path, 0, sizeof(resolved_path));
+
+    if( realpath( relative_path.c_str(), resolved_path) == nullptr ) {
+        std::cerr << "Error resolving path: " << relative_path << std::endl;
+        return "";
+    }
+
+    return std::string(resolved_path);
+}
+

src/misc/helpers.h

@@ -23,6 +23,9 @@
 #define REX_HELPERS_H
 
 #include <string>
+#include <cstring>
+#include <string.h>
+
 #include <sys/stat.h>
 #include <sys/param.h>
 #include <unistd.h>
@@ -51,4 +54,17 @@ std::string get_8601();
 
 const char * command2args( std::string input_string );
 
+/**
+ * @brief Get the absolute path from a relative path
+ *
+ * This function takes a relative path and returns the corresponding absolute path.
+ * The absolute path is obtained by calling the `realpath` function.
+ * If the `realpath` function returns a null pointer, an error message is printed to the standard error stream and an empty string is returned.
+ *
+ * @param relative_path The relative path to be converted to an absolute path
+ *
+ * @return The absolute path corresponding to the relative path
+ */
+std::string get_absolute_path(const std::string &relative_path);
+
 #endif //REX_HELPERS_H

src/plan/Task.cpp

@@ -354,8 +354,19 @@ void Task::execute( Conf * configuration )
     std::string environment_file = this->definition.get_environment_file();
     std::string logs_root = configuration->get_logs_path();
 
+    interpolate(task_name);
     this->slog.log_task( E_DEBUG, task_name, "Using unit definition: \"" + task_name + "\"." );
 
+    interpolate(command);
+    interpolate(shell_name);
+    interpolate(user);
+    interpolate(group);
+    interpolate(environment_file);
+    interpolate(new_working_dir);
+    interpolate(rectifier);
+    interpolate(logs_root);
+
+
     // sanitize all path inputs from unit definition to be either absolute paths or relative to
     // project_root