Interactive input/output using Python
PythonSubprocessStdoutStdinInteractivePython Problem Overview
I have a program that interacts with the user (acts like a shell), and I want to run it using the Python subprocess module interactively. That means, I want the possibility to write to standard input and immediately get the output from standard output. I tried many solutions offered here, but none of them seems to work for my needs.
The code I've written is based on https://stackoverflow.com/questions/11457931/running-an-interactive-command-from-within-python.
import Queue
import threading
import subprocess
def enqueue_output(out, queue):
for line in iter(out.readline, b''):
queue.put(line)
out.close()
def getOutput(outQueue):
outStr = ''
try:
while True: # Adds output from the queue until it is empty
outStr += outQueue.get_nowait()
except Queue.Empty:
return outStr
p = subprocess.Popen("./a.out", stdin=subprocess.PIPE, stout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize = 1)
#p = subprocess.Popen("./a.out", stdin=subprocess.PIPE, stout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, universal_newlines=True)
outQueue = Queue()
errQueue = Queue()
outThread = Thread(target=enqueue_output, args=(p.stdout, outQueue))
errThread = Thread(target=enqueue_output, args=(p.stderr, errQueue))
outThread.daemon = True
errThread.daemon = True
outThread.start()
errThread.start()
p.stdin.write("1\n")
p.stdin.flush()
errors = getOutput(errQueue)
output = getOutput(outQueue)
p.stdin.write("5\n")
p.stdin.flush()
erros = getOutput(errQueue)
output = getOutput(outQueue)
The problem is that the queue remains empty, as if there is no output. Only if I write to standard input all the input that the program needs to execute and terminate, then I get the output (which is not what I want). For example, if I do something like:
p.stdin.write("1\n5\n")
errors = getOutput(errQueue)
output = getOutput(outQueue)
Is there a way to do what I want to do?
The script will run on a Linux machine. I changed my script and deleted the universal_newlines=True + set the bufsize to 1 and flushed standard input immediately after write. Still I don't get any output.
Second try:
I tried this solution and it works for me:
from subprocess import Popen, PIPE
fw = open("tmpout", "wb")
fr = open("tmpout", "r")
p = Popen("./a.out", stdin = PIPE, stdout = fw, stderr = fw, bufsize = 1)
p.stdin.write("1\n")
out = fr.read()
p.stdin.write("5\n")
out = fr.read()
fw.close()
fr.close()
Python Solutions
Solution 1 - Python
None of the current answers worked for me. At the end, I've got this working:
import subprocess
def start(executable_file):
return subprocess.Popen(
executable_file,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
def read(process):
return process.stdout.readline().decode("utf-8").strip()
def write(process, message):
process.stdin.write(f"{message.strip()}\n".encode("utf-8"))
process.stdin.flush()
def terminate(process):
process.stdin.close()
process.terminate()
process.wait(timeout=0.2)
process = start("./dummy.py")
write(process, "hello dummy")
print(read(process))
terminate(process)
Tested with this dummy.py
script:
#!/usr/bin/env python3.6
import random
import time
while True:
message = input()
time.sleep(random.uniform(0.1, 1.0)) # simulates process time
print(message[::-1])
The caveats are (all managed in the functions):
- Input/output always lines with newline.
- Flush child's stdin after every write.
- Use
readline()
from child's stdout.
It's a pretty simple solution in my opinion (not mine, I found it here: https://eli.thegreenplace.net/2017/interacting-with-a-long-running-child-process-in-python/). I was using Python 3.6.
Solution 2 - Python
Two solutions for this issue on Linux:
First one is to use a file to write the output to, and read from it simultaneously:
from subprocess import Popen, PIPE
fw = open("tmpout", "wb")
fr = open("tmpout", "r")
p = Popen("./a.out", stdin = PIPE, stdout = fw, stderr = fw, bufsize = 1)
p.stdin.write("1\n")
out = fr.read()
p.stdin.write("5\n")
out = fr.read()
fw.close()
fr.close()
Second, as J.F. Sebastian offered, is to make p.stdout and p.stderr pipes non-blocking using fnctl module:
import os
import fcntl
from subprocess import Popen, PIPE
def setNonBlocking(fd):
"""
Set the file description of the given file descriptor to non-blocking.
"""
flags = fcntl.fcntl(fd, fcntl.F_GETFL)
flags = flags | os.O_NONBLOCK
fcntl.fcntl(fd, fcntl.F_SETFL, flags)
p = Popen("./a.out", stdin = PIPE, stdout = PIPE, stderr = PIPE, bufsize = 1)
setNonBlocking(p.stdout)
setNonBlocking(p.stderr)
p.stdin.write("1\n")
while True:
try:
out1 = p.stdout.read()
except IOError:
continue
else:
break
out1 = p.stdout.read()
p.stdin.write("5\n")
while True:
try:
out2 = p.stdout.read()
except IOError:
continue
else:
break
Solution 3 - Python
Here is an interactive shell. You have to run read() on a separate thread, otherwise it will block the write()
import sys
import os
import subprocess
from subprocess import Popen, PIPE
import threading
class LocalShell(object):
def __init__(self):
pass
def run(self):
env = os.environ.copy()
p = Popen('/bin/bash', stdin=PIPE, stdout=PIPE, stderr=subprocess.STDOUT, shell=True, env=env)
sys.stdout.write("Started Local Terminal...\r\n\r\n")
def writeall(p):
while True:
# print("read data: ")
data = p.stdout.read(1).decode("utf-8")
if not data:
break
sys.stdout.write(data)
sys.stdout.flush()
writer = threading.Thread(target=writeall, args=(p,))
writer.start()
try:
while True:
d = sys.stdin.read(1)
if not d:
break
self._write(p, d.encode())
except EOFError:
pass
def _write(self, process, message):
process.stdin.write(message)
process.stdin.flush()
shell = LocalShell()
shell.run()
Solution 4 - Python
As I know, the simplest way is to make dedicated threads on both sides: two for stdout/stderr on parent process and one for stdin on child process. Part of such example please find here.