From f6f3210165d2c61a9b135181286fb064a7269d29 Mon Sep 17 00:00:00 2001 From: Hugo Thunnissen Date: Sun, 8 May 2022 10:50:16 +0200 Subject: [PATCH] Implement + test subshell expansion and statement evaluation --- commands/shell/dollar.go | 80 ++++++++++++++++++---------- commands/shell/quotes.go | 10 ++-- commands/shell/quotes_test.go | 32 +++++++++--- commands/shell/state.go | 46 +++++++++++++++++ commands/shell/statement.go | 54 ++++++++++++++++++- commands/shell/statement_test.go | 89 ++++++++++++++++++++++++++++++++ commands/shell/subshell.go | 27 ++++++++++ commands/shell/subshell_test.go | 59 +++++++++++++++++++++ commands/shell/whitespace.go | 6 ++- commands/shell/word.go | 2 +- 10 files changed, 364 insertions(+), 41 deletions(-) create mode 100644 commands/shell/statement_test.go create mode 100644 commands/shell/subshell_test.go diff --git a/commands/shell/dollar.go b/commands/shell/dollar.go index bca3afa..fc720d6 100644 --- a/commands/shell/dollar.go +++ b/commands/shell/dollar.go @@ -1,33 +1,40 @@ package shell import ( + "bytes" "io" "log" ) type DollarParameters struct { ignoreParams *IgnoreFirstCharParameters + subparsers []Parser + lastCreated *Dollar } func (p *DollarParameters) SubParsers() []Parser { - if p.ignoreParams == nil { - p.ignoreParams = &IgnoreFirstCharParameters{} - } + if p.subparsers == nil { + if p.ignoreParams == nil { + p.ignoreParams = &IgnoreFirstCharParameters{} + } - return []Parser{ - &BaseParser{ - parameters: p.ignoreParams, - }, - &BaseParser{ - parameters: &WordParameters{}, - }, - &BaseParser{ - parameters: &SubshellParameters{}, - }, - &BaseParser{ - parameters: &DollarBlockParameters{}, - }, + p.subparsers = []Parser{ + &BaseParser{ + parameters: p.ignoreParams, + }, + &BaseParser{ + parameters: &WordParameters{}, + }, + &BaseParser{ + parameters: &SubshellParameters{}, + }, + &BaseParser{ + parameters: &DollarBlockParameters{}, + }, + } } + + return p.subparsers } func (p *DollarParameters) Enter(_ *CharIterator) error { @@ -48,7 +55,20 @@ func (p *DollarParameters) Supports(charsBefore []rune, r rune) bool { } func (p *DollarParameters) ShouldLeave(charsBefore []rune, r rune) bool { - if isWhitespace(r) || r == ';' { + if p.lastCreated != nil { + if len(p.lastCreated.Tokens()) > 0 { + if _, isSub := p.lastCreated.Tokens()[0].(*Subshell); isSub { + return true + } else if _, isWord := p.lastCreated.Tokens()[0].(*Word); isWord { + return true + } else if _, isBlock := p.lastCreated.Tokens()[0].(*Block); isBlock { + return true + } + + } + } + + if isWhitespace(r) || r == ';' || r == ')' { return true } @@ -60,7 +80,11 @@ func (p *DollarParameters) Leave(i *CharIterator) error { } func (p *DollarParameters) MakeToken() Token { - return &Dollar{} + dollar := &Dollar{} + + p.lastCreated = dollar + + return dollar } type Dollar struct { @@ -82,16 +106,20 @@ func (d *Dollar) Expand(state State, stdin io.Reader, stderr io.Writer) ([]strin if word, ok := block.Tokens()[0].(*Word); ok { return []string{state.Variable(word.String())}, 0 } else { - log.Println("Whaat") + logstr := "Unexpected tokens after dollar sign: " + block.String() + log.Println(logstr) + stderr.Write([]byte(logstr)) + return []string{}, 2 } - } else { - logstr := "Unexpected tokens after dollar sign: " + d.String() - log.Println(logstr) - stderr.Write([]byte(logstr)) - return []string{""}, 2 + } else if subshell, ok := d.Tokens()[0].(*Subshell); ok { + stdout := bytes.NewBuffer([]byte{}) + ret := subshell.Evaluate(state, stdin, stdout, stderr) + return []string{stdout.String()}, ret } - - return []string{""}, 1 + logstr := "Unexpected tokens after dollar sign: " + d.String() + log.Println(logstr) + stderr.Write([]byte(logstr)) + return []string{""}, 2 } type IgnoreFirstCharParameters struct { diff --git a/commands/shell/quotes.go b/commands/shell/quotes.go index fe5ee0a..c396cb1 100644 --- a/commands/shell/quotes.go +++ b/commands/shell/quotes.go @@ -2,7 +2,6 @@ package shell import ( "io" - "log" "strings" ) @@ -19,6 +18,10 @@ func (p *QuoteParameters) SubParsers() []Parser { } } +func (p *QuoteParameters) Enter(i *CharIterator) error { + return i.Next() +} + func (p *QuoteParameters) Supports(charsBefore []rune, r rune) bool { if r == '"' { return len(charsBefore) == 0 || @@ -51,18 +54,17 @@ func (q *Quote) Expand(state State, stdin io.Reader, stderr io.Writer) ([]string for _, token := range q.Tokens() { if expand, ok := token.(Expandable); ok { - log.Println(expand) var expansion []string expansion, retcode = expand.Expand(state, stdin, stderr) - str += strings.Join(expansion, " ") + str += strings.Join(expansion, "") continue } str += token.String() } - return []string{strings.Trim(str, "'")}, retcode + return []string{str}, retcode } type SingleQuoteParameters struct { diff --git a/commands/shell/quotes_test.go b/commands/shell/quotes_test.go index 2aa556a..00bee62 100644 --- a/commands/shell/quotes_test.go +++ b/commands/shell/quotes_test.go @@ -3,7 +3,6 @@ package shell import ( "bufio" "fmt" - "io" "strings" "testing" @@ -18,16 +17,14 @@ type QuotesTest struct { func (t *QuotesTest) TestParse() { expectedContents := fmt.Sprintf("'%s'", "word1 word2 word3\n word4") - parser := &BaseParser{&QuoteParameters{}} + parser := &BaseParser{&SingleQuoteParameters{}} reader := bufio.NewReader(strings.NewReader(expectedContents)) iterator, err := NewCharIterator(reader) t.NoError(err) token, err := parser.Parse(iterator, &CharCollection{}) - t.ErrorIs(err, io.EOF) - - t.Equal(expectedContents, token.String()) + t.NoError(err) expanded, _ := token.(Expandable).Expand(nil, nil, nil) t.Equal("word1 word2 word3\n word4", strings.Join(expanded, " ")) @@ -36,13 +33,31 @@ func (t *QuotesTest) TestParse() { func (t *QuotesTest) TestExpandVariables() { parser := &BaseParser{&QuoteParameters{}} reader := bufio.NewReader( - strings.NewReader("This is a $variable ${variable2} $variable"), + strings.NewReader("\"This is a $variable ${variable2} $variable\""), ) state := NewShellState() state.SetVariable("variable", "supergood") state.SetVariable("variable2", "thing") + iterator, err := NewCharIterator(reader) + t.NoError(err) + + token, err := parser.Parse(iterator, &CharCollection{}) + t.NoError(err) + + expanded, _ := token.(Expandable).Expand(state, nil, nil) + t.Equal("This is a supergood thing supergood", expanded[0]) +} + +func (t *QuotesTest) TestExpandSubshell() { + parser := &BaseParser{&QuoteParameters{}} + reader := bufio.NewReader( + strings.NewReader("\"This is $(echo a subshell)\""), + ) + + state := NewShellState() + echo := &builtins.Echo{} state.SetCommand(echo.Name(), echo) @@ -50,10 +65,11 @@ func (t *QuotesTest) TestExpandVariables() { t.NoError(err) token, err := parser.Parse(iterator, &CharCollection{}) - t.ErrorIs(err, io.EOF) + t.NoError(err) expanded, _ := token.(Expandable).Expand(state, nil, nil) - t.Equal("This is a supergood thing supergood", expanded[0]) + t.Equal(1, len(expanded)) + t.Equal("This is a subshell", expanded[0]) } func TestQuotesTest(t *testing.T) { diff --git a/commands/shell/state.go b/commands/shell/state.go index 64e2d67..beba924 100644 --- a/commands/shell/state.go +++ b/commands/shell/state.go @@ -1,5 +1,12 @@ package shell +import ( + "fmt" + "io" + "log" + "strconv" +) + type State interface { Variable(string) string Command(string) CommandExecutor @@ -13,6 +20,11 @@ type State interface { SetCommand(string, CommandExecutor) SetEnv(string, string) + Eval(Evalable, io.Reader, io.Writer, io.Writer) uint8 + Expand(Expandable, io.Reader, io.Writer) []string + + ExecuteCommand(string, []string, io.Reader, io.Writer, io.Writer) uint8 + Clone() State } @@ -110,3 +122,37 @@ func (s *ShellState) Clone() State { return state } + +func (s *ShellState) Eval(e Evalable, stdin io.Reader, stdout io.Writer, stderr io.Writer) uint8 { + ret := e.Evaluate(s, stdin, stdout, stderr) + + s.SetVariable("?", strconv.Itoa(int(ret))) + + return ret +} + +func (s *ShellState) Expand(e Expandable, stdin io.Reader, stderr io.Writer) []string { + out, ret := e.Expand(s, stdin, stderr) + + s.SetVariable("?", strconv.Itoa(int(ret))) + + return out +} + +func (s *ShellState) ExecuteCommand( + name string, + args []string, + stdin io.Reader, + stdout, + stderr io.Writer, +) uint8 { + if comm, ok := s.commands[name]; ok { + return comm.Execute(args, stdin, stdout, stderr) + } + + logstr := fmt.Sprintf("Shell: command %s not found", name) + log.Println(logstr) + stderr.Write([]byte(logstr)) + + return 127 +} diff --git a/commands/shell/statement.go b/commands/shell/statement.go index 5603b83..db588cf 100644 --- a/commands/shell/statement.go +++ b/commands/shell/statement.go @@ -1,9 +1,50 @@ package shell +import ( + "io" + "log" +) + type Statement struct { BaseToken } +func (s *Statement) Evaluate(state State, stdin io.Reader, stdout io.Writer, stderr io.Writer) uint8 { + if len(s.Tokens()) == 0 { + return 0 + } + + var command string + var arguments []string + for _, token := range s.Tokens() { + var values []string + + if expand, ok := token.(Expandable); ok { + values = state.Expand(expand, stdin, stdout) + } else if _, isWhitespace := token.(*Whitespace); isWhitespace { + continue + } else { + logstr := "shell: Unexpected token: " + token.String() + log.Println(logstr) + stderr.Write([]byte(logstr)) + + return 2 + } + + if command == "" { + command = values[0] + if len(values) > 1 { + arguments = values[1:] + } + continue + } + + arguments = append(arguments, values...) + } + + return state.ExecuteCommand(command, arguments, stdin, stdout, stderr) +} + type StatementParameters struct{} func (p *StatementParameters) SubParsers() []Parser { @@ -29,6 +70,9 @@ func (p *StatementParameters) SubParsers() []Parser { &BaseParser{ parameters: &BlockParameters{}, }, + &BaseParser{ + parameters: &WhitespaceParameters{}, + }, } } @@ -41,9 +85,17 @@ func (p *StatementParameters) MakeToken() Token { } func (p *StatementParameters) ShouldLeave(charsBefore []rune, r rune) bool { - if r == '\n' || r == ';' { + if r == '\n' || r == ';' || r == ')' { return countBackslashSuffixes(charsBefore)%2 == 0 } return false } + +func (p *StatementParameters) Leave(i *CharIterator) error { + if i.Current() == '\n' || i.Current() == ')' { + return i.Previous() + } + + return nil +} diff --git a/commands/shell/statement_test.go b/commands/shell/statement_test.go new file mode 100644 index 0000000..648ee9c --- /dev/null +++ b/commands/shell/statement_test.go @@ -0,0 +1,89 @@ +package shell + +import ( + "bufio" + "bytes" + "io" + "strings" + "testing" + + "git.snorba.art/hugo/nssh/commands/shell/builtins" + "github.com/stretchr/testify/suite" +) + +type StatementTest struct { + suite.Suite +} + +func (t *StatementTest) TestParse() { + parser := &BaseParser{&StatementParameters{}} + reader := bufio.NewReader(strings.NewReader("echo hello world!")) + + iterator, err := NewCharIterator(reader) + t.NoError(err) + + statement, err := parser.Parse(iterator, &CharCollection{}) + t.ErrorIs(err, io.EOF) + + t.IsType(&Word{}, statement.Tokens()[0]) + t.IsType(&Whitespace{}, statement.Tokens()[1]) + t.IsType(&Word{}, statement.Tokens()[2]) + t.IsType(&Whitespace{}, statement.Tokens()[3]) + t.IsType(&Word{}, statement.Tokens()[4]) + + t.Equal(5, len(statement.Tokens())) + + t.Equal("echo", statement.Tokens()[0].String()) + t.Equal(" ", statement.Tokens()[1].String()) + t.Equal("hello", statement.Tokens()[2].String()) + t.Equal(" ", statement.Tokens()[3].String()) + t.Equal("world!", statement.Tokens()[4].String()) +} + +func (t *StatementTest) TestEval() { + parser := &BaseParser{&StatementParameters{}} + reader := bufio.NewReader(strings.NewReader("echo hello world!")) + + iterator, err := NewCharIterator(reader) + t.NoError(err) + + statement, err := parser.Parse(iterator, &CharCollection{}) + t.ErrorIs(err, io.EOF) + + out := []byte{} + outBuf := bytes.NewBuffer(out) + state := NewShellState() + + echo := &builtins.Echo{} + state.SetCommand(echo.Name(), echo) + + state.Eval(statement.(Evalable), nil, outBuf, nil) + + t.Equal("hello world!", string(outBuf.Bytes())) +} + +func (t *StatementTest) TestEvalWithSubshell() { + parser := &BaseParser{&StatementParameters{}} + reader := bufio.NewReader(strings.NewReader("echo \"$(echo hello world!)\"")) + + iterator, err := NewCharIterator(reader) + t.NoError(err) + + statement, err := parser.Parse(iterator, &CharCollection{}) + t.ErrorIs(err, io.EOF) + + out := []byte{} + outBuf := bytes.NewBuffer(out) + state := NewShellState() + + echo := &builtins.Echo{} + state.SetCommand(echo.Name(), echo) + + state.Eval(statement.(Evalable), nil, outBuf, nil) + + t.Equal("hello world!", string(outBuf.Bytes())) +} + +func TestStatementTest(t *testing.T) { + suite.Run(t, new(StatementTest)) +} diff --git a/commands/shell/subshell.go b/commands/shell/subshell.go index 1e9f4be..99c5246 100644 --- a/commands/shell/subshell.go +++ b/commands/shell/subshell.go @@ -1,5 +1,10 @@ package shell +import ( + "io" + "log" +) + type SubshellParameters struct{} func (p *SubshellParameters) SubParsers() []Parser { @@ -10,6 +15,10 @@ func (p *SubshellParameters) SubParsers() []Parser { } } +func (p *SubshellParameters) Enter(i *CharIterator) error { + return i.Next() +} + func (p *SubshellParameters) Supports(charsBefore []rune, r rune) bool { if r == '(' { return len(charsBefore) == 0 || @@ -36,6 +45,24 @@ type Subshell struct { BaseToken } +func (s *Subshell) Evaluate(state State, stdin io.Reader, stdout io.Writer, stderr io.Writer) uint8 { + var ret uint8 + + for _, token := range s.Tokens() { + if eval, ok := token.(Evalable); ok { + ret = state.Eval(eval, stdin, stdout, stderr) + } else { + logstr := "shell: Unexpected token: " + token.String() + log.Println(logstr) + stderr.Write([]byte(logstr)) + + return 2 + } + } + + return ret +} + type Backtick struct { Subshell } diff --git a/commands/shell/subshell_test.go b/commands/shell/subshell_test.go new file mode 100644 index 0000000..7ca539d --- /dev/null +++ b/commands/shell/subshell_test.go @@ -0,0 +1,59 @@ +package shell + +import ( + "bufio" + "strings" + "testing" + + "github.com/stretchr/testify/suite" +) + +type SubshellTest struct { + suite.Suite +} + +func (t *SubshellTest) TestParse() { + parser := &BaseParser{&SubshellParameters{}} + reader := bufio.NewReader(strings.NewReader("(echo hello world!)")) + + iterator, err := NewCharIterator(reader) + t.NoError(err) + + token, err := parser.Parse(iterator, &CharCollection{}) + t.NoError(err) + + t.IsType(&Subshell{}, token) + t.IsType(&Statement{}, token.Tokens()[0]) + t.Equal(1, len(token.Tokens())) + + statement := token.Tokens()[0] + t.IsType(&Word{}, statement.Tokens()[0]) + t.IsType(&Word{}, statement.Tokens()[2]) + t.IsType(&Word{}, statement.Tokens()[4]) + + t.Equal(5, len(statement.Tokens())) + + t.Equal("echo", statement.Tokens()[0].String()) + t.Equal("hello", statement.Tokens()[2].String()) + t.Equal("world!", statement.Tokens()[4].String()) +} + +func (t *SubshellTest) TestParseMultipleStatements() { + parser := &BaseParser{&SubshellParameters{}} + reader := bufio.NewReader(strings.NewReader("(echo hello world!; echo tramp_exit_code $?)")) + + iterator, err := NewCharIterator(reader) + t.NoError(err) + + token, err := parser.Parse(iterator, &CharCollection{}) + t.NoError(err) + + t.Equal(2, len(token.Tokens())) + t.IsType(&Subshell{}, token) + t.IsType(&Statement{}, token.Tokens()[0]) + t.IsType(&Statement{}, token.Tokens()[1]) +} + +func TestSubshellTest(t *testing.T) { + suite.Run(t, new(SubshellTest)) +} diff --git a/commands/shell/whitespace.go b/commands/shell/whitespace.go index cf4072b..13254dc 100644 --- a/commands/shell/whitespace.go +++ b/commands/shell/whitespace.go @@ -5,7 +5,7 @@ type WhitespaceParameters struct{} func (p *WhitespaceParameters) SubParsers() []Parser { return []Parser{ &BaseParser{ - parameters: &WhitespaceParameters{}, + parameters: &CharParserParameters{}, }, } } @@ -22,6 +22,10 @@ func (p *WhitespaceParameters) MakeToken() Token { return &Whitespace{} } +func (p *WhitespaceParameters) Leave(i *CharIterator) error { + return i.Previous() +} + type Whitespace struct { tokens []Token } diff --git a/commands/shell/word.go b/commands/shell/word.go index a3e657d..53f4de3 100644 --- a/commands/shell/word.go +++ b/commands/shell/word.go @@ -68,7 +68,7 @@ func (p *WordParameters) MakeToken() Token { return new(Word) } -var wordRegexp = regexp.MustCompile("[0-9a-zA-Z_.~/:\\\\=-]") +var wordRegexp = regexp.MustCompile("[0-9a-zA-Z_.~/:\\\\!?*=-]") func (p *WordParameters) Supports(charsBefore []rune, r rune) bool { return wordRegexp.MatchString(string(r))