How to pipe several commands in Go?
GoPipeGo Problem Overview
How can I pipe several external commands together in Go? I've tried this code but I get an error that says exit status 1.
package main
import (
"io"
"log"
"os"
"os/exec"
)
func main() {
c1 := exec.Command("ls")
stdout1, err := c1.StdoutPipe()
if err != nil {
log.Fatal(err)
}
if err = c1.Start(); err != nil {
log.Fatal(err)
}
if err = c1.Wait(); err != nil {
log.Fatal(err)
}
c2 := exec.Command("wc", "-l")
c2.Stdin = stdout1
stdout2, err := c2.StdoutPipe()
if err != nil {
log.Fatal(err)
}
if err = c2.Start(); err != nil {
log.Fatal(err)
}
if err = c2.Wait(); err != nil {
log.Fatal(err)
}
io.Copy(os.Stdout, stdout2)
}
Go Solutions
Solution 1 - Go
For simple scenarios, you could use this approach:
bash -c "echo 'your command goes here'"
For example, this function retrieves the CPU model name using piped commands:
func getCPUmodel() string {
cmd := "cat /proc/cpuinfo | egrep '^model name' | uniq | awk '{print substr($0, index($0,$4))}'"
out, err := exec.Command("bash","-c",cmd).Output()
if err != nil {
return fmt.Sprintf("Failed to execute command: %s", cmd)
}
return string(out)
}
Solution 2 - Go
> StdoutPipe returns a pipe that will be connected to the command's > standard output when the command starts. The pipe will be closed > automatically after Wait sees the command exit.
(from http://golang.org/pkg/os/exec/#Cmd.StdinPipe )
The fact you do c1.Wait closes the stdoutPipe.
I made a working example (just a demo, add error catching!) :
package main
import (
"bytes"
"io"
"os"
"os/exec"
)
func main() {
c1 := exec.Command("ls")
c2 := exec.Command("wc", "-l")
r, w := io.Pipe()
c1.Stdout = w
c2.Stdin = r
var b2 bytes.Buffer
c2.Stdout = &b2
c1.Start()
c2.Start()
c1.Wait()
w.Close()
c2.Wait()
io.Copy(os.Stdout, &b2)
}
Solution 3 - Go
package main
import (
"os"
"os/exec"
)
func main() {
c1 := exec.Command("ls")
c2 := exec.Command("wc", "-l")
c2.Stdin, _ = c1.StdoutPipe()
c2.Stdout = os.Stdout
_ = c2.Start()
_ = c1.Run()
_ = c2.Wait()
}
Solution 4 - Go
Like the first answer but with the first command started and waited for in a goroutine. This keeps the pipe happy.
package main
import (
"io"
"os"
"os/exec"
)
func main() {
c1 := exec.Command("ls")
c2 := exec.Command("wc", "-l")
pr, pw := io.Pipe()
c1.Stdout = pw
c2.Stdin = pr
c2.Stdout = os.Stdout
c1.Start()
c2.Start()
go func() {
defer pw.Close()
c1.Wait()
}()
c2.Wait()
}
Solution 5 - Go
This is a fully working example. The Execute function takes any number of exec.Cmd instances (using a variadic function) and then loops over them correctly attaching the output of stdout to the stdin of the next command. This must be done before any function is called.
The call function then goes about calling the commands in a loop, using defers to call recursively and ensuring proper closure of pipes
package main
import (
"bytes"
"io"
"log"
"os"
"os/exec"
)
func Execute(output_buffer *bytes.Buffer, stack ...*exec.Cmd) (err error) {
var error_buffer bytes.Buffer
pipe_stack := make([]*io.PipeWriter, len(stack)-1)
i := 0
for ; i < len(stack)-1; i++ {
stdin_pipe, stdout_pipe := io.Pipe()
stack[i].Stdout = stdout_pipe
stack[i].Stderr = &error_buffer
stack[i+1].Stdin = stdin_pipe
pipe_stack[i] = stdout_pipe
}
stack[i].Stdout = output_buffer
stack[i].Stderr = &error_buffer
if err := call(stack, pipe_stack); err != nil {
log.Fatalln(string(error_buffer.Bytes()), err)
}
return err
}
func call(stack []*exec.Cmd, pipes []*io.PipeWriter) (err error) {
if stack[0].Process == nil {
if err = stack[0].Start(); err != nil {
return err
}
}
if len(stack) > 1 {
if err = stack[1].Start(); err != nil {
return err
}
defer func() {
if err == nil {
pipes[0].Close()
err = call(stack[1:], pipes[1:])
}
}()
}
return stack[0].Wait()
}
func main() {
var b bytes.Buffer
if err := Execute(&b,
exec.Command("ls", "/Users/tyndyll/Downloads"),
exec.Command("grep", "as"),
exec.Command("sort", "-r"),
); err != nil {
log.Fatalln(err)
}
io.Copy(os.Stdout, &b)
}
Available in this gist
https://gist.github.com/tyndyll/89fbb2c2273f83a074dc
A good point to know is that shell variables like ~ are not interpolated
Solution 6 - Go
I wanted to pipe some video and audio to FFplay. This worked for me:
package main
import (
"io"
"os/exec"
)
func main() {
ffmpeg := exec.Command(
"ffmpeg", "-i", "247.webm", "-i", "251.webm", "-c", "copy", "-f", "webm", "-",
)
ffplay := exec.Command("ffplay", "-")
ffplay.Stdin, ffmpeg.Stdout = io.Pipe()
ffmpeg.Start()
ffplay.Run()
}
Solution 7 - Go
package main
import (
...
pipe "github.com/b4b4r07/go-pipe"
)
func main() {
var b bytes.Buffer
pipe.Command(&b,
exec.Command("ls", "/Users/b4b4r07/Downloads"),
exec.Command("grep", "Vim"),
)
io.Copy(os.Stdout, &b)
}
I spent a good day trying to use Denys Séguret answer to come up with a wrapper for multiple exec.Command before I came across this neat package by b4b4r07.
Solution 8 - Go
Because it can be complex to build such command chains I have decided to implements a litte go library for that purpose: https://github.com/rainu/go-command-chain
package main
import (
"bytes"
"fmt"
"github.com/rainu/go-command-chain"
)
func main() {
output := &bytes.Buffer{}
err := cmdchain.Builder().
Join("ls").
Join("wc", "-l").
Finalize().WithOutput(output).Run()
if err != nil {
panic(err)
}
fmt.Printf("Errors found: %s", output)
}
With the help of this lib you can also configure std-error forwarding and other things.