subprocess.h raw
1 // Based on the https://github.com/arun11299/cpp-subprocess project.
2
3 /*!
4
5 Documentation for C++ subprocessing library.
6
7 @copyright The code is licensed under the [MIT
8 License](http://opensource.org/licenses/MIT):
9 <br>
10 Copyright © 2016-2018 Arun Muralidharan.
11 <br>
12 Permission is hereby granted, free of charge, to any person obtaining a copy
13 of this software and associated documentation files (the "Software"), to deal
14 in the Software without restriction, including without limitation the rights
15 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
16 copies of the Software, and to permit persons to whom the Software is
17 furnished to do so, subject to the following conditions:
18 <br>
19 The above copyright notice and this permission notice shall be included in
20 all copies or substantial portions of the Software.
21 <br>
22 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
23 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
25 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
27 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
28 SOFTWARE.
29
30 @author [Arun Muralidharan]
31 @see https://github.com/arun11299/cpp-subprocess to download the source code
32
33 @version 1.0.0
34 */
35
36 #ifndef BITCOIN_UTIL_SUBPROCESS_H
37 #define BITCOIN_UTIL_SUBPROCESS_H
38
39 #include <util/check.h>
40 #include <util/syserror.h>
41
42 #include <algorithm>
43 #include <csignal>
44 #include <cstdio>
45 #include <cstdlib>
46 #include <cstring>
47 #include <exception>
48 #include <future>
49 #include <initializer_list>
50 #include <iostream>
51 #include <locale>
52 #include <map>
53 #include <memory>
54 #include <sstream>
55 #include <string>
56 #include <vector>
57
58 #ifdef WIN32
59 #include <codecvt>
60 #endif
61
62 extern "C" {
63 #ifdef WIN32
64 #include <windows.h>
65 #include <io.h>
66 #include <cwchar>
67 #else
68 #include <sys/wait.h>
69 #include <unistd.h>
70 #endif
71 #include <csignal>
72 #include <fcntl.h>
73 #include <sys/types.h>
74 }
75
76 // The Microsoft C++ compiler issues deprecation warnings
77 // for the standard POSIX function names.
78 // Its preferred implementations have a leading underscore.
79 // See: https://learn.microsoft.com/en-us/cpp/c-runtime-library/compatibility.
80 #if (defined _MSC_VER)
81 #define subprocess_close _close
82 #define subprocess_fileno _fileno
83 #define subprocess_open _open
84 #define subprocess_write _write
85 #else
86 #define subprocess_close close
87 #define subprocess_fileno fileno
88 #define subprocess_open open
89 #define subprocess_write write
90 #endif
91
92 /*!
93 * Getting started with reading this source code.
94 * The source is mainly divided into four parts:
95 * 1. Exception Classes:
96 * These are very basic exception classes derived from
97 * runtime_error exception.
98 * There are two types of exception thrown from subprocess
99 * library: OSError and CalledProcessError
100 *
101 * 2. Popen Class
102 * This is the main class the users will deal with. It
103 * provides with all the API's to deal with processes.
104 *
105 * 3. Util namespace
106 * It includes some helper functions to split/join a string,
107 * reading from file descriptors, waiting on a process, fcntl
108 * options on file descriptors etc.
109 *
110 * 4. Detail namespace
111 * This includes some metaprogramming and helper classes.
112 */
113
114
115 namespace subprocess {
116
117 // Max buffer size allocated on stack for read error
118 // from pipe
119 static const size_t SP_MAX_ERR_BUF_SIZ = 1024;
120
121 // Default buffer capacity for OutBuffer and ErrBuffer.
122 // If the data exceeds this capacity, the buffer size is grown
123 // by 1.5 times its previous capacity
124 static const size_t DEFAULT_BUF_CAP_BYTES = 8192;
125
126
127 /*-----------------------------------------------
128 * EXCEPTION CLASSES
129 *-----------------------------------------------
130 */
131
132 /*!
133 * class: CalledProcessError
134 * Thrown when there was error executing the command.
135 * Check Popen class API's to know when this exception
136 * can be thrown.
137 *
138 */
139 class CalledProcessError: public std::runtime_error
140 {
141 public:
142 int retcode;
143 CalledProcessError(const std::string& error_msg, int retcode):
144 std::runtime_error(error_msg), retcode(retcode)
145 {}
146 };
147
148
149 /*!
150 * class: OSError
151 * Thrown when some system call fails to execute or give result.
152 * The exception message contains the name of the failed system call
153 * with the stringisized errno code.
154 * Check Popen class API's to know when this exception would be
155 * thrown.
156 * Its usual that the API exception specification would have
157 * this exception together with CalledProcessError.
158 */
159 class OSError: public std::runtime_error
160 {
161 public:
162 OSError(const std::string& err_msg, int err_code):
163 std::runtime_error(err_msg + ": " + SysErrorString(err_code))
164 {}
165 };
166
167 //--------------------------------------------------------------------
168 namespace util
169 {
170 #ifdef WIN32
171 inline void quote_argument(const std::wstring &argument, std::wstring &command_line,
172 bool force)
173 {
174 //
175 // Unless we're told otherwise, don't quote unless we actually
176 // need to do so --- hopefully avoid problems if programs won't
177 // parse quotes properly
178 //
179
180 if (force == false && argument.empty() == false &&
181 argument.find_first_of(L" \t\n\v") == argument.npos) {
182 command_line.append(argument);
183 }
184 else {
185 command_line.push_back(L'"');
186
187 for (auto it = argument.begin();; ++it) {
188 unsigned number_backslashes = 0;
189
190 while (it != argument.end() && *it == L'\\') {
191 ++it;
192 ++number_backslashes;
193 }
194
195 if (it == argument.end()) {
196
197 //
198 // Escape all backslashes, but let the terminating
199 // double quotation mark we add below be interpreted
200 // as a metacharacter.
201 //
202
203 command_line.append(number_backslashes * 2, L'\\');
204 break;
205 }
206 else if (*it == L'"') {
207
208 //
209 // Escape all backslashes and the following
210 // double quotation mark.
211 //
212
213 command_line.append(number_backslashes * 2 + 1, L'\\');
214 command_line.push_back(*it);
215 }
216 else {
217
218 //
219 // Backslashes aren't special here.
220 //
221
222 command_line.append(number_backslashes, L'\\');
223 command_line.push_back(*it);
224 }
225 }
226
227 command_line.push_back(L'"');
228 }
229 }
230
231 inline std::string get_last_error(DWORD errorMessageID)
232 {
233 if (errorMessageID == 0)
234 return std::string();
235
236 LPSTR messageBuffer = nullptr;
237 size_t size = FormatMessageA(
238 FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM |
239 FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_MAX_WIDTH_MASK,
240 NULL, errorMessageID, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
241 (LPSTR)&messageBuffer, 0, NULL);
242
243 std::string message(messageBuffer, size);
244
245 LocalFree(messageBuffer);
246
247 return message;
248 }
249
250 inline FILE *file_from_handle(HANDLE h, const char *mode)
251 {
252 int md;
253 if (!mode) {
254 throw OSError("invalid_mode", 0);
255 }
256
257 if (mode[0] == 'w') {
258 md = _O_WRONLY;
259 }
260 else if (mode[0] == 'r') {
261 md = _O_RDONLY;
262 }
263 else {
264 throw OSError("file_from_handle", 0);
265 }
266
267 int os_fhandle = _open_osfhandle((intptr_t)h, md);
268 if (os_fhandle == -1) {
269 CloseHandle(h);
270 throw OSError("_open_osfhandle", 0);
271 }
272
273 FILE *fp = _fdopen(os_fhandle, mode);
274 if (fp == 0) {
275 subprocess_close(os_fhandle);
276 throw OSError("_fdopen", 0);
277 }
278
279 return fp;
280 }
281
282 inline void configure_pipe(HANDLE* read_handle, HANDLE* write_handle, HANDLE* child_handle)
283 {
284 SECURITY_ATTRIBUTES saAttr;
285
286 // Set the bInheritHandle flag so pipe handles are inherited.
287 saAttr.nLength = sizeof(SECURITY_ATTRIBUTES);
288 saAttr.bInheritHandle = TRUE;
289 saAttr.lpSecurityDescriptor = NULL;
290
291 // Create a pipe for the child process's STDIN.
292 if (!CreatePipe(read_handle, write_handle, &saAttr,0))
293 throw OSError("CreatePipe", 0);
294
295 // Ensure the write handle to the pipe for STDIN is not inherited.
296 if (!SetHandleInformation(*child_handle, HANDLE_FLAG_INHERIT, 0))
297 throw OSError("SetHandleInformation", 0);
298 }
299 #endif
300
301 /*!
302 * Function: split
303 * Parameters:
304 * [in] str : Input string which needs to be split based upon the
305 * delimiters provided.
306 * [in] deleims : Delimiter characters based upon which the string needs
307 * to be split. Default constructed to ' '(space) and '\t'(tab)
308 * [out] vector<string> : Vector of strings split at deleimiter.
309 */
310 static inline std::vector<std::string>
311 split(const std::string& str, const std::string& delims=" \t")
312 {
313 std::vector<std::string> res;
314 size_t init = 0;
315
316 while (true) {
317 auto pos = str.find_first_of(delims, init);
318 if (pos == std::string::npos) {
319 res.emplace_back(str.substr(init, str.length()));
320 break;
321 }
322 res.emplace_back(str.substr(init, pos - init));
323 pos++;
324 init = pos;
325 }
326
327 return res;
328 }
329
330
331 #ifndef WIN32
332 /*!
333 * Function: set_clo_on_exec
334 * Sets/Resets the FD_CLOEXEC flag on the provided file descriptor
335 * based upon the `set` parameter.
336 * Parameters:
337 * [in] fd : The descriptor on which FD_CLOEXEC needs to be set/reset.
338 * [in] set : If 'true', set FD_CLOEXEC.
339 * If 'false' unset FD_CLOEXEC.
340 */
341 static inline
342 void set_clo_on_exec(int fd, bool set = true)
343 {
344 int flags = fcntl(fd, F_GETFD, 0);
345 if (flags == -1) {
346 throw OSError("fcntl F_GETFD failed", errno);
347 }
348 if (set) flags |= FD_CLOEXEC;
349 else flags &= ~FD_CLOEXEC;
350 if (fcntl(fd, F_SETFD, flags) == -1) {
351 throw OSError("fcntl F_SETFD failed", errno);
352 }
353 }
354
355
356 /*!
357 * Function: pipe_cloexec
358 * Creates a pipe and sets FD_CLOEXEC flag on both
359 * read and write descriptors of the pipe.
360 * Parameters:
361 * [out] : A pair of file descriptors.
362 * First element of pair is the read descriptor of pipe.
363 * Second element is the write descriptor of pipe.
364 */
365 static inline
366 std::pair<int, int> pipe_cloexec() noexcept(false)
367 {
368 int pipe_fds[2];
369 int res = pipe(pipe_fds);
370 if (res) {
371 throw OSError("pipe failure", errno);
372 }
373
374 set_clo_on_exec(pipe_fds[0]);
375 set_clo_on_exec(pipe_fds[1]);
376
377 return std::make_pair(pipe_fds[0], pipe_fds[1]);
378 }
379 #endif
380
381
382 /*!
383 * Function: write_n
384 * Writes `length` bytes to the file descriptor `fd`
385 * from the buffer `buf`.
386 * Parameters:
387 * [in] fd : The file descriptotr to write to.
388 * [in] buf: Buffer from which data needs to be written to fd.
389 * [in] length: The number of bytes that needs to be written from
390 * `buf` to `fd`.
391 * [out] int : Number of bytes written or -1 in case of failure.
392 */
393 static inline
394 int write_n(int fd, const char* buf, size_t length)
395 {
396 size_t nwritten = 0;
397 while (nwritten < length) {
398 int written = subprocess_write(fd, buf + nwritten, length - nwritten);
399 if (written == -1) return -1;
400 nwritten += written;
401 }
402 return nwritten;
403 }
404
405
406 /*!
407 * Function: read_atmost_n
408 * Reads at the most `read_upto` bytes from the
409 * file object `fp` before returning.
410 * Parameters:
411 * [in] fp : The file object from which it needs to read.
412 * [in] buf : The buffer into which it needs to write the data.
413 * [in] read_upto: Max number of bytes which must be read from `fd`.
414 * [out] int : Number of bytes written to `buf` or read from `fd`
415 * OR -1 in case of error.
416 * NOTE: In case of EINTR while reading from socket, this API
417 * will retry to read from `fd`, but only till the EINTR counter
418 * reaches 50 after which it will return with whatever data it read.
419 */
420 static inline
421 int read_atmost_n(FILE* fp, char* buf, size_t read_upto)
422 {
423 #ifdef WIN32
424 return (int)fread(buf, 1, read_upto, fp);
425 #else
426 int fd = subprocess_fileno(fp);
427 int rbytes = 0;
428 int eintr_cnter = 0;
429
430 while (1) {
431 int read_bytes = read(fd, buf + rbytes, read_upto - rbytes);
432 if (read_bytes == -1) {
433 if (errno == EINTR) {
434 if (eintr_cnter >= 50) return -1;
435 eintr_cnter++;
436 continue;
437 }
438 return -1;
439 }
440 if (read_bytes == 0) return rbytes;
441
442 rbytes += read_bytes;
443 }
444 return rbytes;
445 #endif
446 }
447
448
449 /*!
450 * Function: read_all
451 * Reads all the available data from `fp` into
452 * `buf`. Internally calls read_atmost_n.
453 * Parameters:
454 * [in] fp : The file object from which to read from.
455 * [in] buf : The buffer of type `class Buffer` into which
456 * the read data is written to.
457 * [out] int: Number of bytes read OR -1 in case of failure.
458 *
459 * NOTE: `class Buffer` is a exposed public class. See below.
460 */
461
462 static inline int read_all(FILE* fp, std::vector<char>& buf)
463 {
464 auto buffer = buf.data();
465 int total_bytes_read = 0;
466 int fill_sz = buf.size();
467
468 while (1) {
469 const int rd_bytes = read_atmost_n(fp, buffer, fill_sz);
470
471 if (rd_bytes == -1) { // Read finished
472 if (total_bytes_read == 0) return -1;
473 break;
474
475 } else if (rd_bytes == fill_sz) { // Buffer full
476 const auto orig_sz = buf.size();
477 const auto new_sz = orig_sz * 2;
478 buf.resize(new_sz);
479 fill_sz = new_sz - orig_sz;
480
481 //update the buffer pointer
482 buffer = buf.data();
483 total_bytes_read += rd_bytes;
484 buffer += total_bytes_read;
485
486 } else { // Partial data ? Continue reading
487 total_bytes_read += rd_bytes;
488 fill_sz -= rd_bytes;
489 break;
490 }
491 }
492 buf.erase(buf.begin()+total_bytes_read, buf.end()); // remove extra nulls
493 return total_bytes_read;
494 }
495
496 #ifndef WIN32
497 /*!
498 * Function: wait_for_child_exit
499 * Waits for the process with pid `pid` to exit
500 * and returns its status.
501 * Parameters:
502 * [in] pid : The pid of the process.
503 * [out] pair<int, int>:
504 * pair.first : Return code of the waitpid call.
505 * pair.second : Exit status of the process.
506 *
507 * NOTE: This is a blocking call as in, it will loop
508 * till the child is exited.
509 */
510 static inline
511 std::pair<int, int> wait_for_child_exit(int pid)
512 {
513 int status = 0;
514 int ret = -1;
515 while (1) {
516 ret = waitpid(pid, &status, 0);
517 if (ret == -1) break;
518 if (ret == 0) continue;
519 return std::make_pair(ret, status);
520 }
521
522 return std::make_pair(ret, status);
523 }
524 #endif
525
526 } // end namespace util
527
528
529
530 /* -------------------------------
531 * Popen Arguments
532 * -------------------------------
533 */
534
535 /*!
536 * Base class for all arguments involving string value.
537 */
538 struct string_arg
539 {
540 string_arg(const char* arg): arg_value(arg) {}
541 string_arg(std::string&& arg): arg_value(std::move(arg)) {}
542 string_arg(const std::string& arg): arg_value(arg) {}
543 std::string arg_value;
544 };
545
546 /*!
547 * Option to specify the executable name separately
548 * from the args sequence.
549 * In this case the cmd args must only contain the
550 * options required for this executable.
551 *
552 * Eg: executable{"ls"}
553 */
554 struct executable: string_arg
555 {
556 template <typename T>
557 executable(T&& arg): string_arg(std::forward<T>(arg)) {}
558 };
559
560 /*!
561 * Used for redirecting input/output/error
562 */
563 enum IOTYPE {
564 STDOUT = 1,
565 STDERR,
566 PIPE,
567 };
568
569 //TODO: A common base/interface for below stream structures ??
570
571 /*!
572 * Option to specify the input channel for the child
573 * process. It can be:
574 * 1. An already open file descriptor.
575 * 2. A file name.
576 * 3. IOTYPE. Usual a PIPE
577 *
578 * Eg: input{PIPE}
579 * OR in case of redirection, output of another Popen
580 * input{popen.output()}
581 */
582 struct input
583 {
584 // For an already existing file descriptor.
585 explicit input(int fd): rd_ch_(fd) {}
586
587 // FILE pointer.
588 explicit input (FILE* fp):input(subprocess_fileno(fp)) { assert(fp); }
589
590 explicit input(const char* filename) {
591 int fd = subprocess_open(filename, O_RDONLY);
592 if (fd == -1) throw OSError("File not found: ", errno);
593 rd_ch_ = fd;
594 }
595 explicit input(IOTYPE typ) {
596 assert (typ == PIPE && "STDOUT/STDERR not allowed");
597 #ifndef WIN32
598 std::tie(rd_ch_, wr_ch_) = util::pipe_cloexec();
599 #endif
600 }
601
602 int rd_ch_ = -1;
603 int wr_ch_ = -1;
604 };
605
606
607 /*!
608 * Option to specify the output channel for the child
609 * process. It can be:
610 * 1. An already open file descriptor.
611 * 2. A file name.
612 * 3. IOTYPE. Usually a PIPE.
613 *
614 * Eg: output{PIPE}
615 * OR output{"output.txt"}
616 */
617 struct output
618 {
619 explicit output(int fd): wr_ch_(fd) {}
620
621 explicit output (FILE* fp):output(subprocess_fileno(fp)) { assert(fp); }
622
623 explicit output(const char* filename) {
624 int fd = subprocess_open(filename, O_APPEND | O_CREAT | O_RDWR, 0640);
625 if (fd == -1) throw OSError("File not found: ", errno);
626 wr_ch_ = fd;
627 }
628 explicit output(IOTYPE typ) {
629 assert (typ == PIPE && "STDOUT/STDERR not allowed");
630 #ifndef WIN32
631 std::tie(rd_ch_, wr_ch_) = util::pipe_cloexec();
632 #endif
633 }
634
635 int rd_ch_ = -1;
636 int wr_ch_ = -1;
637 };
638
639
640 /*!
641 * Option to specify the error channel for the child
642 * process. It can be:
643 * 1. An already open file descriptor.
644 * 2. A file name.
645 * 3. IOTYPE. Usually a PIPE or STDOUT
646 *
647 */
648 struct error
649 {
650 explicit error(int fd): wr_ch_(fd) {}
651
652 explicit error(FILE* fp):error(subprocess_fileno(fp)) { assert(fp); }
653
654 explicit error(const char* filename) {
655 int fd = subprocess_open(filename, O_APPEND | O_CREAT | O_RDWR, 0640);
656 if (fd == -1) throw OSError("File not found: ", errno);
657 wr_ch_ = fd;
658 }
659 explicit error(IOTYPE typ) {
660 assert ((typ == PIPE || typ == STDOUT) && "STDERR not allowed");
661 if (typ == PIPE) {
662 #ifndef WIN32
663 std::tie(rd_ch_, wr_ch_) = util::pipe_cloexec();
664 #endif
665 } else {
666 // Need to defer it till we have checked all arguments
667 deferred_ = true;
668 }
669 }
670
671 bool deferred_ = false;
672 int rd_ch_ = -1;
673 int wr_ch_ = -1;
674 };
675
676 // ~~~~ End Popen Args ~~~~
677
678
679 /*!
680 * class: Buffer
681 * This class is a very thin wrapper around std::vector<char>
682 * This is basically used to determine the length of the actual
683 * data stored inside the dynamically resized vector.
684 *
685 * This is what is returned as the output to the communicate
686 * function, so, users must know about this class.
687 *
688 * OutBuffer and ErrBuffer are just different typedefs to this class.
689 */
690 class Buffer
691 {
692 public:
693 Buffer() = default;
694 explicit Buffer(size_t cap) { buf.resize(cap); }
695 void add_cap(size_t cap) { buf.resize(cap); }
696
697 public:
698 std::vector<char> buf;
699 size_t length = 0;
700 };
701
702 // Buffer for storing output written to output fd
703 using OutBuffer = Buffer;
704 // Buffer for storing output written to error fd
705 using ErrBuffer = Buffer;
706
707
708 // Fwd Decl.
709 class Popen;
710
711 /*---------------------------------------------------
712 * DETAIL NAMESPACE
713 *---------------------------------------------------
714 */
715
716 namespace detail {
717 /*!
718 * A helper class to Popen class for setting
719 * options as provided in the Popen constructor.
720 * This design allows us to _not_ have any fixed position
721 * to any arguments and specify them in a way similar to what
722 * can be done in python.
723 */
724 struct ArgumentDeducer
725 {
726 ArgumentDeducer(Popen* p): popen_(p) {}
727
728 void set_option(executable&& exe);
729 void set_option(input&& inp);
730 void set_option(output&& out);
731 void set_option(error&& err);
732
733 private:
734 Popen* popen_ = nullptr;
735 };
736
737 #ifndef WIN32
738 /*!
739 * A helper class to Popen.
740 * This takes care of all the fork-exec logic
741 * in the execute_child API.
742 */
743 class Child
744 {
745 public:
746 Child(Popen* p, int err_wr_pipe):
747 parent_(p),
748 err_wr_pipe_(err_wr_pipe)
749 {}
750
751 void execute_child();
752
753 private:
754 // Lets call it parent even though
755 // technically a bit incorrect
756 Popen* parent_ = nullptr;
757 int err_wr_pipe_ = -1;
758 };
759 #endif
760
761 // Fwd Decl.
762 class Streams;
763
764 /*!
765 * A helper class to Streams.
766 * This takes care of management of communicating
767 * with the child process with the means of the correct
768 * file descriptor.
769 */
770 class Communication
771 {
772 public:
773 Communication(Streams* stream): stream_(stream)
774 {}
775 Communication(const Communication&) = delete;
776 Communication& operator=(const Communication&) = delete;
777 Communication(Communication&&) = default;
778 Communication& operator=(Communication&&) = default;
779 public:
780 int send(const char* msg, size_t length);
781 int send(const std::vector<char>& msg);
782
783 std::pair<OutBuffer, ErrBuffer> communicate(const char* msg, size_t length);
784 std::pair<OutBuffer, ErrBuffer> communicate(const std::vector<char>& msg)
785 { return communicate(msg.data(), msg.size()); }
786
787 void set_out_buf_cap(size_t cap) { out_buf_cap_ = cap; }
788 void set_err_buf_cap(size_t cap) { err_buf_cap_ = cap; }
789
790 private:
791 std::pair<OutBuffer, ErrBuffer> communicate_threaded(
792 const char* msg, size_t length);
793
794 private:
795 Streams* stream_;
796 size_t out_buf_cap_ = DEFAULT_BUF_CAP_BYTES;
797 size_t err_buf_cap_ = DEFAULT_BUF_CAP_BYTES;
798 };
799
800
801
802 /*!
803 * This is a helper class to Popen.
804 * It takes care of management of all the file descriptors
805 * and file pointers.
806 * It dispatches of the communication aspects to the
807 * Communication class.
808 * Read through the data members to understand about the
809 * various file descriptors used.
810 */
811 class Streams
812 {
813 public:
814 Streams():comm_(this) {}
815 Streams(const Streams&) = delete;
816 Streams& operator=(const Streams&) = delete;
817 Streams(Streams&&) = default;
818 Streams& operator=(Streams&&) = default;
819
820 public:
821 void setup_comm_channels();
822
823 void cleanup_fds()
824 {
825 if (write_to_child_ != -1 && read_from_parent_ != -1) {
826 subprocess_close(write_to_child_);
827 }
828 if (write_to_parent_ != -1 && read_from_child_ != -1) {
829 subprocess_close(read_from_child_);
830 }
831 if (err_write_ != -1 && err_read_ != -1) {
832 subprocess_close(err_read_);
833 }
834 }
835
836 void close_parent_fds()
837 {
838 if (write_to_child_ != -1) subprocess_close(write_to_child_);
839 if (read_from_child_ != -1) subprocess_close(read_from_child_);
840 if (err_read_ != -1) subprocess_close(err_read_);
841 }
842
843 void close_child_fds()
844 {
845 if (write_to_parent_ != -1) subprocess_close(write_to_parent_);
846 if (read_from_parent_ != -1) subprocess_close(read_from_parent_);
847 if (err_write_ != -1) subprocess_close(err_write_);
848 }
849
850 FILE* input() { return input_.get(); }
851 FILE* output() { return output_.get(); }
852 FILE* error() { return error_.get(); }
853
854 void input(FILE* fp) { input_.reset(fp, fclose); }
855 void output(FILE* fp) { output_.reset(fp, fclose); }
856 void error(FILE* fp) { error_.reset(fp, fclose); }
857
858 void set_out_buf_cap(size_t cap) { comm_.set_out_buf_cap(cap); }
859 void set_err_buf_cap(size_t cap) { comm_.set_err_buf_cap(cap); }
860
861 public: /* Communication forwarding API's */
862 int send(const char* msg, size_t length)
863 { return comm_.send(msg, length); }
864
865 int send(const std::vector<char>& msg)
866 { return comm_.send(msg); }
867
868 std::pair<OutBuffer, ErrBuffer> communicate(const char* msg, size_t length)
869 { return comm_.communicate(msg, length); }
870
871 std::pair<OutBuffer, ErrBuffer> communicate(const std::vector<char>& msg)
872 { return comm_.communicate(msg); }
873
874
875 public:// Yes they are public
876
877 std::shared_ptr<FILE> input_ = nullptr;
878 std::shared_ptr<FILE> output_ = nullptr;
879 std::shared_ptr<FILE> error_ = nullptr;
880
881 #ifdef WIN32
882 HANDLE g_hChildStd_IN_Rd = nullptr;
883 HANDLE g_hChildStd_IN_Wr = nullptr;
884 HANDLE g_hChildStd_OUT_Rd = nullptr;
885 HANDLE g_hChildStd_OUT_Wr = nullptr;
886 HANDLE g_hChildStd_ERR_Rd = nullptr;
887 HANDLE g_hChildStd_ERR_Wr = nullptr;
888 #endif
889
890 // Pipes for communicating with child
891
892 // Emulates stdin
893 int write_to_child_ = -1; // Parent owned descriptor
894 int read_from_parent_ = -1; // Child owned descriptor
895
896 // Emulates stdout
897 int write_to_parent_ = -1; // Child owned descriptor
898 int read_from_child_ = -1; // Parent owned descriptor
899
900 // Emulates stderr
901 int err_write_ = -1; // Write error to parent (Child owned)
902 int err_read_ = -1; // Read error from child (Parent owned)
903
904 private:
905 Communication comm_;
906 };
907
908 } // end namespace detail
909
910
911
912 /*!
913 * class: Popen
914 * This is the single most important class in the whole library
915 * and glues together all the helper classes to provide a common
916 * interface to the client.
917 *
918 * API's provided by the class:
919 * Popen({"cmd"}, output{..}, error{..}, ....)
920 * Command provided as a sequence.
921 * wait() - Wait for the child to exit.
922 * retcode() - The return code of the exited child.
923 * send(...) - Send input to the input channel of the child.
924 * communicate(...) - Get the output/error from the child and close the channels
925 * from the parent side.
926 */
927 class Popen
928 {
929 public:
930 friend struct detail::ArgumentDeducer;
931 #ifndef WIN32
932 friend class detail::Child;
933 #endif
934
935 template <typename... Args>
936 Popen(std::initializer_list<const char*> cmd_args, Args&& ...args)
937 {
938 vargs_.insert(vargs_.end(), cmd_args.begin(), cmd_args.end());
939 init_args(std::forward<Args>(args)...);
940
941 // Setup the communication channels of the Popen class
942 stream_.setup_comm_channels();
943
944 execute_process();
945 }
946
947 template <typename... Args>
948 Popen(std::vector<std::string> vargs_, Args &&... args) : vargs_(vargs_)
949 {
950 init_args(std::forward<Args>(args)...);
951
952 // Setup the communication channels of the Popen class
953 stream_.setup_comm_channels();
954
955 execute_process();
956 }
957
958 int retcode() const noexcept { return retcode_; }
959
960 int wait() noexcept(false);
961
962 void set_out_buf_cap(size_t cap) { stream_.set_out_buf_cap(cap); }
963
964 void set_err_buf_cap(size_t cap) { stream_.set_err_buf_cap(cap); }
965
966 int send(const char* msg, size_t length)
967 { return stream_.send(msg, length); }
968
969 int send(const std::string& msg)
970 { return send(msg.c_str(), msg.size()); }
971
972 int send(const std::vector<char>& msg)
973 { return stream_.send(msg); }
974
975 std::pair<OutBuffer, ErrBuffer> communicate(const char* msg, size_t length)
976 {
977 auto res = stream_.communicate(msg, length);
978 retcode_ = wait();
979 return res;
980 }
981
982 std::pair<OutBuffer, ErrBuffer> communicate(const std::string& msg)
983 {
984 return communicate(msg.c_str(), msg.size());
985 }
986
987 std::pair<OutBuffer, ErrBuffer> communicate(const std::vector<char>& msg)
988 {
989 auto res = stream_.communicate(msg);
990 retcode_ = wait();
991 return res;
992 }
993
994 std::pair<OutBuffer, ErrBuffer> communicate()
995 {
996 return communicate(nullptr, 0);
997 }
998
999 private:
1000 template <typename F, typename... Args>
1001 void init_args(F&& farg, Args&&... args);
1002 void init_args();
1003 void populate_c_argv();
1004 void execute_process() noexcept(false);
1005
1006 private:
1007 detail::Streams stream_;
1008
1009 #ifdef WIN32
1010 HANDLE process_handle_;
1011 std::future<void> cleanup_future_;
1012 #else
1013 // Pid of the child process
1014 int child_pid_ = -1;
1015 #endif
1016
1017 std::string exe_name_;
1018
1019 // Command provided as sequence
1020 std::vector<std::string> vargs_;
1021 std::vector<char*> cargv_;
1022
1023 int retcode_ = -1;
1024 };
1025
1026 inline void Popen::init_args() {
1027 populate_c_argv();
1028 }
1029
1030 template <typename F, typename... Args>
1031 inline void Popen::init_args(F&& farg, Args&&... args)
1032 {
1033 detail::ArgumentDeducer argd(this);
1034 argd.set_option(std::forward<F>(farg));
1035 init_args(std::forward<Args>(args)...);
1036 }
1037
1038 inline void Popen::populate_c_argv()
1039 {
1040 cargv_.clear();
1041 cargv_.reserve(vargs_.size() + 1);
1042 for (auto& arg : vargs_) cargv_.push_back(&arg[0]);
1043 cargv_.push_back(nullptr);
1044 }
1045
1046 inline int Popen::wait() noexcept(false)
1047 {
1048 #ifdef WIN32
1049 int ret = WaitForSingleObject(process_handle_, INFINITE);
1050
1051 // WaitForSingleObject with INFINITE should only return when process has signaled
1052 if (ret != WAIT_OBJECT_0) {
1053 throw OSError("Unexpected return code from WaitForSingleObject", 0);
1054 }
1055
1056 DWORD dretcode_;
1057
1058 if (FALSE == GetExitCodeProcess(process_handle_, &dretcode_))
1059 throw OSError("Failed during call to GetExitCodeProcess", 0);
1060
1061 CloseHandle(process_handle_);
1062
1063 return (int)dretcode_;
1064 #else
1065 int ret, status;
1066 std::tie(ret, status) = util::wait_for_child_exit(child_pid_);
1067 if (ret == -1) {
1068 if (errno != ECHILD) throw OSError("waitpid failed", errno);
1069 return 0;
1070 }
1071 if (WIFEXITED(status)) return WEXITSTATUS(status);
1072 if (WIFSIGNALED(status)) return WTERMSIG(status);
1073 else return 255;
1074
1075 return 0;
1076 #endif
1077 }
1078
1079 inline void Popen::execute_process() noexcept(false)
1080 {
1081 #ifdef WIN32
1082 if (exe_name_.length()) {
1083 this->vargs_.insert(this->vargs_.begin(), this->exe_name_);
1084 this->populate_c_argv();
1085 }
1086 this->exe_name_ = vargs_[0];
1087
1088 std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
1089 std::wstring argument;
1090 std::wstring command_line;
1091 bool first_arg = true;
1092
1093 for (auto arg : this->vargs_) {
1094 if (!first_arg) {
1095 command_line += L" ";
1096 } else {
1097 first_arg = false;
1098 }
1099 argument = converter.from_bytes(arg);
1100 util::quote_argument(argument, command_line, false);
1101 }
1102
1103 // CreateProcessW can modify szCmdLine so we allocate needed memory
1104 wchar_t *szCmdline = new wchar_t[command_line.size() + 1];
1105 wcscpy_s(szCmdline, command_line.size() + 1, command_line.c_str());
1106 PROCESS_INFORMATION piProcInfo;
1107 STARTUPINFOW siStartInfo;
1108 BOOL bSuccess = FALSE;
1109 DWORD creation_flags = CREATE_UNICODE_ENVIRONMENT | CREATE_NO_WINDOW;
1110
1111 // Set up members of the PROCESS_INFORMATION structure.
1112 ZeroMemory(&piProcInfo, sizeof(PROCESS_INFORMATION));
1113
1114 // Set up members of the STARTUPINFOW structure.
1115 // This structure specifies the STDIN and STDOUT handles for redirection.
1116
1117 ZeroMemory(&siStartInfo, sizeof(STARTUPINFOW));
1118 siStartInfo.cb = sizeof(STARTUPINFOW);
1119
1120 siStartInfo.hStdError = this->stream_.g_hChildStd_ERR_Wr;
1121 siStartInfo.hStdOutput = this->stream_.g_hChildStd_OUT_Wr;
1122 siStartInfo.hStdInput = this->stream_.g_hChildStd_IN_Rd;
1123
1124 siStartInfo.dwFlags |= STARTF_USESTDHANDLES;
1125
1126 // Create the child process.
1127 bSuccess = CreateProcessW(NULL,
1128 szCmdline, // command line
1129 NULL, // process security attributes
1130 NULL, // primary thread security attributes
1131 TRUE, // handles are inherited
1132 creation_flags, // creation flags
1133 NULL, // use parent's environment
1134 NULL, // use parent's current directory
1135 &siStartInfo, // STARTUPINFOW pointer
1136 &piProcInfo); // receives PROCESS_INFORMATION
1137
1138 // If an error occurs, exit the application.
1139 if (!bSuccess) {
1140 DWORD errorMessageID = ::GetLastError();
1141 throw CalledProcessError("CreateProcess failed: " + util::get_last_error(errorMessageID), errorMessageID);
1142 }
1143
1144 CloseHandle(piProcInfo.hThread);
1145
1146 /*
1147 TODO: use common apis to close linux handles
1148 */
1149
1150 this->process_handle_ = piProcInfo.hProcess;
1151
1152 this->cleanup_future_ = std::async(std::launch::async, [this] {
1153 WaitForSingleObject(this->process_handle_, INFINITE);
1154
1155 CloseHandle(this->stream_.g_hChildStd_ERR_Wr);
1156 CloseHandle(this->stream_.g_hChildStd_OUT_Wr);
1157 CloseHandle(this->stream_.g_hChildStd_IN_Rd);
1158 });
1159
1160 /*
1161 NOTE: In the linux version, there is a check to make sure that the process
1162 has been started. Here, we do nothing because CreateProcess will throw
1163 if we fail to create the process.
1164 */
1165
1166
1167 #else
1168
1169 int err_rd_pipe, err_wr_pipe;
1170 std::tie(err_rd_pipe, err_wr_pipe) = util::pipe_cloexec();
1171
1172 if (exe_name_.length()) {
1173 vargs_.insert(vargs_.begin(), exe_name_);
1174 populate_c_argv();
1175 }
1176 exe_name_ = vargs_[0];
1177
1178 child_pid_ = fork();
1179
1180 if (child_pid_ < 0) {
1181 subprocess_close(err_rd_pipe);
1182 subprocess_close(err_wr_pipe);
1183 throw OSError("fork failed", errno);
1184 }
1185
1186 if (child_pid_ == 0)
1187 {
1188 // Close descriptors belonging to parent
1189 stream_.close_parent_fds();
1190
1191 //Close the read end of the error pipe
1192 subprocess_close(err_rd_pipe);
1193
1194 detail::Child chld(this, err_wr_pipe);
1195 chld.execute_child();
1196 }
1197 else
1198 {
1199 subprocess_close(err_wr_pipe);// close child side of pipe, else get stuck in read below
1200
1201 stream_.close_child_fds();
1202
1203 try {
1204 char err_buf[SP_MAX_ERR_BUF_SIZ] = {0,};
1205
1206 FILE* err_fp = fdopen(err_rd_pipe, "r");
1207 if (!err_fp) {
1208 subprocess_close(err_rd_pipe);
1209 throw OSError("fdopen failed", errno);
1210 }
1211 int read_bytes = util::read_atmost_n(err_fp, err_buf, SP_MAX_ERR_BUF_SIZ);
1212 fclose(err_fp);
1213
1214 if (read_bytes || strlen(err_buf)) {
1215 // Call waitpid to reap the child process
1216 // waitpid suspends the calling process until the
1217 // child terminates.
1218 int retcode = wait();
1219
1220 // Throw whatever information we have about child failure
1221 throw CalledProcessError(err_buf, retcode);
1222 }
1223 } catch (std::exception& exp) {
1224 stream_.cleanup_fds();
1225 throw;
1226 }
1227
1228 }
1229 #endif
1230 }
1231
1232 namespace detail {
1233
1234 inline void ArgumentDeducer::set_option(executable&& exe) {
1235 popen_->exe_name_ = std::move(exe.arg_value);
1236 }
1237
1238 inline void ArgumentDeducer::set_option(input&& inp) {
1239 if (inp.rd_ch_ != -1) popen_->stream_.read_from_parent_ = inp.rd_ch_;
1240 if (inp.wr_ch_ != -1) popen_->stream_.write_to_child_ = inp.wr_ch_;
1241 }
1242
1243 inline void ArgumentDeducer::set_option(output&& out) {
1244 if (out.wr_ch_ != -1) popen_->stream_.write_to_parent_ = out.wr_ch_;
1245 if (out.rd_ch_ != -1) popen_->stream_.read_from_child_ = out.rd_ch_;
1246 }
1247
1248 inline void ArgumentDeducer::set_option(error&& err) {
1249 if (err.deferred_) {
1250 if (popen_->stream_.write_to_parent_) {
1251 popen_->stream_.err_write_ = popen_->stream_.write_to_parent_;
1252 } else {
1253 throw std::runtime_error("Set output before redirecting error to output");
1254 }
1255 }
1256 if (err.wr_ch_ != -1) popen_->stream_.err_write_ = err.wr_ch_;
1257 if (err.rd_ch_ != -1) popen_->stream_.err_read_ = err.rd_ch_;
1258 }
1259
1260
1261 #ifndef WIN32
1262 inline void Child::execute_child() {
1263 int sys_ret = -1;
1264 auto& stream = parent_->stream_;
1265
1266 try {
1267 if (stream.write_to_parent_ == 0)
1268 stream.write_to_parent_ = dup(stream.write_to_parent_);
1269
1270 if (stream.err_write_ == 0 || stream.err_write_ == 1)
1271 stream.err_write_ = dup(stream.err_write_);
1272
1273 // Make the child owned descriptors as the
1274 // stdin, stdout and stderr for the child process
1275 auto _dup2_ = [](int fd, int to_fd) {
1276 if (fd == to_fd) {
1277 // dup2 syscall does not reset the
1278 // CLOEXEC flag if the descriptors
1279 // provided to it are same.
1280 // But, we need to reset the CLOEXEC
1281 // flag as the provided descriptors
1282 // are now going to be the standard
1283 // input, output and error
1284 util::set_clo_on_exec(fd, false);
1285 } else if(fd != -1) {
1286 int res = dup2(fd, to_fd);
1287 if (res == -1) throw OSError("dup2 failed", errno);
1288 }
1289 };
1290
1291 // Create the standard streams
1292 _dup2_(stream.read_from_parent_, 0); // Input stream
1293 _dup2_(stream.write_to_parent_, 1); // Output stream
1294 _dup2_(stream.err_write_, 2); // Error stream
1295
1296 // Close the duped descriptors
1297 if (stream.read_from_parent_ != -1 && stream.read_from_parent_ > 2)
1298 subprocess_close(stream.read_from_parent_);
1299
1300 if (stream.write_to_parent_ != -1 && stream.write_to_parent_ > 2)
1301 subprocess_close(stream.write_to_parent_);
1302
1303 if (stream.err_write_ != -1 && stream.err_write_ > 2)
1304 subprocess_close(stream.err_write_);
1305
1306 // Replace the current image with the executable
1307 sys_ret = execvp(parent_->exe_name_.c_str(), parent_->cargv_.data());
1308
1309 if (sys_ret == -1) throw OSError("execve failed", errno);
1310
1311 } catch (const OSError& exp) {
1312 // Just write the exception message
1313 // TODO: Give back stack trace ?
1314 std::string err_msg(exp.what());
1315 //ATTN: Can we do something on error here ?
1316 util::write_n(err_wr_pipe_, err_msg.c_str(), err_msg.length());
1317 }
1318
1319 // Calling application would not get this
1320 // exit failure
1321 _exit (EXIT_FAILURE);
1322 }
1323 #endif
1324
1325
1326 inline void Streams::setup_comm_channels()
1327 {
1328 #ifdef WIN32
1329 util::configure_pipe(&this->g_hChildStd_IN_Rd, &this->g_hChildStd_IN_Wr, &this->g_hChildStd_IN_Wr);
1330 this->input(util::file_from_handle(this->g_hChildStd_IN_Wr, "w"));
1331 this->write_to_child_ = subprocess_fileno(this->input());
1332
1333 util::configure_pipe(&this->g_hChildStd_OUT_Rd, &this->g_hChildStd_OUT_Wr, &this->g_hChildStd_OUT_Rd);
1334 this->output(util::file_from_handle(this->g_hChildStd_OUT_Rd, "r"));
1335 this->read_from_child_ = subprocess_fileno(this->output());
1336
1337 util::configure_pipe(&this->g_hChildStd_ERR_Rd, &this->g_hChildStd_ERR_Wr, &this->g_hChildStd_ERR_Rd);
1338 this->error(util::file_from_handle(this->g_hChildStd_ERR_Rd, "r"));
1339 this->err_read_ = subprocess_fileno(this->error());
1340 #else
1341
1342 if (write_to_child_ != -1) input(fdopen(write_to_child_, "wb"));
1343 if (read_from_child_ != -1) output(fdopen(read_from_child_, "rb"));
1344 if (err_read_ != -1) error(fdopen(err_read_, "rb"));
1345
1346 auto handles = {input(), output(), error()};
1347
1348 for (auto& h : handles) {
1349 if (h == nullptr) continue;
1350 setvbuf(h, nullptr, _IONBF, BUFSIZ);
1351 }
1352 #endif
1353 }
1354
1355 inline int Communication::send(const char* msg, size_t length)
1356 {
1357 if (stream_->input() == nullptr) return -1;
1358 return std::fwrite(msg, sizeof(char), length, stream_->input());
1359 }
1360
1361 inline int Communication::send(const std::vector<char>& msg)
1362 {
1363 return send(msg.data(), msg.size());
1364 }
1365
1366 inline std::pair<OutBuffer, ErrBuffer>
1367 Communication::communicate(const char* msg, size_t length)
1368 {
1369 // Optimization from subprocess.py
1370 // If we are using one pipe, or no pipe
1371 // at all, using select() or threads is unnecessary.
1372 auto hndls = {stream_->input(), stream_->output(), stream_->error()};
1373 int count = std::count(std::begin(hndls), std::end(hndls), nullptr);
1374 const int len_conv = length;
1375
1376 if (count >= 2) {
1377 OutBuffer obuf;
1378 ErrBuffer ebuf;
1379 if (stream_->input()) {
1380 if (msg) {
1381 int wbytes = std::fwrite(msg, sizeof(char), length, stream_->input());
1382 if (wbytes < len_conv) {
1383 if (errno != EPIPE && errno != EINVAL) {
1384 throw OSError("fwrite error", errno);
1385 }
1386 }
1387 }
1388 // Close the input stream
1389 stream_->input_.reset();
1390 } else if (stream_->output()) {
1391 // Read till EOF
1392 // ATTN: This could be blocking, if the process
1393 // at the other end screws up, we get screwed as well
1394 obuf.add_cap(out_buf_cap_);
1395
1396 int rbytes = util::read_all(
1397 stream_->output(),
1398 obuf.buf);
1399
1400 if (rbytes == -1) {
1401 throw OSError("read to obuf failed", errno);
1402 }
1403
1404 obuf.length = rbytes;
1405 // Close the output stream
1406 stream_->output_.reset();
1407
1408 } else if (stream_->error()) {
1409 // Same screwness applies here as well
1410 ebuf.add_cap(err_buf_cap_);
1411
1412 int rbytes = util::read_atmost_n(
1413 stream_->error(),
1414 ebuf.buf.data(),
1415 ebuf.buf.size());
1416
1417 if (rbytes == -1) {
1418 throw OSError("read to ebuf failed", errno);
1419 }
1420
1421 ebuf.length = rbytes;
1422 // Close the error stream
1423 stream_->error_.reset();
1424 }
1425 return std::make_pair(std::move(obuf), std::move(ebuf));
1426 }
1427
1428 return communicate_threaded(msg, length);
1429 }
1430
1431
1432 inline std::pair<OutBuffer, ErrBuffer>
1433 Communication::communicate_threaded(const char* msg, size_t length)
1434 {
1435 OutBuffer obuf;
1436 ErrBuffer ebuf;
1437 std::future<int> out_fut, err_fut;
1438 const int length_conv = length;
1439
1440 if (stream_->output()) {
1441 obuf.add_cap(out_buf_cap_);
1442
1443 out_fut = std::async(std::launch::async,
1444 [&obuf, this] {
1445 return util::read_all(this->stream_->output(), obuf.buf);
1446 });
1447 }
1448 if (stream_->error()) {
1449 ebuf.add_cap(err_buf_cap_);
1450
1451 err_fut = std::async(std::launch::async,
1452 [&ebuf, this] {
1453 return util::read_all(this->stream_->error(), ebuf.buf);
1454 });
1455 }
1456 if (stream_->input()) {
1457 if (msg) {
1458 int wbytes = std::fwrite(msg, sizeof(char), length, stream_->input());
1459 if (wbytes < length_conv) {
1460 if (errno != EPIPE && errno != EINVAL) {
1461 throw OSError("fwrite error", errno);
1462 }
1463 }
1464 }
1465 stream_->input_.reset();
1466 }
1467
1468 if (out_fut.valid()) {
1469 int res = out_fut.get();
1470 if (res != -1) obuf.length = res;
1471 else obuf.length = 0;
1472 }
1473 if (err_fut.valid()) {
1474 int res = err_fut.get();
1475 if (res != -1) ebuf.length = res;
1476 else ebuf.length = 0;
1477 }
1478
1479 return std::make_pair(std::move(obuf), std::move(ebuf));
1480 }
1481
1482 } // end namespace detail
1483
1484 }
1485
1486 #endif // BITCOIN_UTIL_SUBPROCESS_H
1487