├── .gitignore
├── BackgroundProcesses.md
├── BackgroundProcessesRevisited.md
├── Environment.md
├── Globbing.md
├── Makefile
├── Piping.md
├── Prompts.md
├── README.md
├── TabCompletion.md
├── TabCompletionRevisited.md
├── Tokenization.md
├── completion.go
├── goshrc
├── main.go
├── prefix.go
├── prefix_test.go
├── tokenize.go
└── tokenize_test.go


/.gitignore:
--------------------------------------------------------------------------------
1 | gosh


--------------------------------------------------------------------------------
/BackgroundProcesses.md:
--------------------------------------------------------------------------------
  1 | # Background Processes and Signals
  2 | 
  3 | To be really useful as a shell, we need to at least handle the
  4 | basics of background processes. Let's start with starting a process in
  5 | the background.
  6 | 
  7 | This should be easy[1]. All we need to do is check if the last token
  8 | in the command is an `&`, and if it is call `cmd.Start()` instead of
  9 | `cmd.Wait()` in our command handler. In fact, we should probably
 10 | start by ensuring we treat '&' as a special token when tokenizing.
 11 | 
 12 | ([1] NB: The below is wrong, both from a theoretical, and a bugginess
 13 | perspective. See BackgroundProcessesRevisited.md for a (more) accurate
 14 | implementation of background/foreground process coordination. This remains
 15 | as an exploration of how someone who's ignorant of the details of UNIX process
 16 | group arcana (which I was, at the time of writing) might approach the problem.)
 17 | 
 18 | Recall we had:
 19 | 
 20 | ### "Handle Tokenize Chr"
 21 | ```go
 22 | switch chr {
 23 | 	case '\'':
 24 | 		<<<Handle Quote>>>
 25 | 	case '|':
 26 | 		<<<Handle Pipe Chr>>>
 27 | 	default:
 28 | 		<<<Handle Nonquote>>>
 29 | }
 30 | ```
 31 | 
 32 | While we're at it, let's treat `<` and `>` specially to make our
 33 | redirection a little more robust. They all follow more or less
 34 | the same rule: be a delimiting token, unless it's inside a string.
 35 | 
 36 | ### "Handle Tokenize Chr"
 37 | ```go
 38 | switch chr {
 39 | 	case '\'':
 40 | 		<<<Handle Quote>>>
 41 | 	case '|':
 42 | 		<<<Handle Pipe Chr>>>
 43 | 	default:
 44 | 		<<<Handle Nonquote>>>
 45 | }
 46 | ```
 47 | 
 48 | 
 49 | Handle Pipe Chr looked like this:
 50 | 
 51 | ### "Handle Pipe Chr"
 52 | ```go
 53 | if inStringLiteral {
 54 | 	continue
 55 | }
 56 | if tokenStart >= 0 {
 57 | 	parsed = append(parsed, string(c[tokenStart:i]))
 58 | }
 59 | parsed = append(parsed, "|")
 60 | tokenStart = -1
 61 | ```
 62 | 
 63 | If we change that from append "|" to append string(chr), we should
 64 | be able to handle them all in the same case that would look like this:
 65 | 
 66 | ### "Handle Tokenize Chr"
 67 | ```go
 68 | switch chr {
 69 | 	case '\'':
 70 | 		<<<Handle Quote>>>
 71 | 	case '|', '<', '>', '&':
 72 | 		<<<Handle Special Chr>>>
 73 | 	default:
 74 | 		<<<Handle Nonquote>>>
 75 | }
 76 | ```
 77 | 
 78 | And then:
 79 | 
 80 | ### "Handle Special Chr"
 81 | ```go
 82 | if inStringLiteral {
 83 | 	continue
 84 | }
 85 | if tokenStart >= 0 {
 86 | 	parsed = append(parsed, string(c[tokenStart:i]))
 87 | }
 88 | parsed = append(parsed, string(chr))
 89 | tokenStart = -1
 90 | ```
 91 | 
 92 | Now, we're ready to just check if the command ends in '&' and use
 93 | `Start` instead of `Wait` if so.
 94 | 
 95 | ### "HandleCmd Implementation"
 96 | ```go
 97 | func (c Command) HandleCmd() error {
 98 | 	parsed := c.Tokenize()
 99 | 	<<<Handle no tokens in command case>>>
100 | 	<<<Replace environment variables in command>>>
101 | 	<<<Handle cd command>>>
102 | 	<<<Execute command and return>>>
103 | }
104 | ```
105 | 
106 | becomes
107 | 
108 | ### "HandleCmd Implementation"
109 | ```go
110 | func (c Command) HandleCmd() error {
111 | 	parsed := c.Tokenize()
112 | 	<<<Handle no tokens in command case>>>
113 | 	<<<Replace environment variables in command>>>
114 | 	<<<Handle background inputs>>>
115 | 	<<<Handle cd command>>>
116 | 	<<<Execute command and return>>>
117 | }
118 | ```
119 | 
120 | where "Handle background inputs" just needs to check if the
121 | parsed tokens ends in a `&`, and if so set a flag for `Start Processes
122 | and Wait` to skip the waiting bit.
123 | 
124 | ### "Handle background inputs"
125 | ```go
126 | var backgroundProcess bool
127 | if parsed[len(parsed)-1] == "&" {
128 | 	parsed = parsed[:len(parsed)-1]
129 | 	backgroundProcess = true
130 | }
131 | ```
132 | 
133 | ### "Start Processes and Wait"
134 | ```go
135 | for _, c := range cmds {
136 | 	err := c.Start()
137 | 	if err != nil {
138 | 		fmt.Fprintf(os.Stderr, "%v\n", err)
139 | 	}
140 | }
141 | if backgroundProcess {
142 | 	// We can't tell if a background process returns an error
143 | 	// or not, so we just claim it didn't.
144 | 	return nil
145 | }
146 | return cmds[len(cmds)-1].Wait()
147 | ```
148 | 
149 | Now when we try running a long-running command like `more BackgroundProcsses.md`
150 | for testing, it doesn't quite work how we exepcted. *Both* the background
151 | process and the shell are hooked up to STDIN.
152 | 
153 | While in `zsh` we get output like
154 | 
155 | ```sh
156 | $ more BackgroundProcsses.md&
157 | [2] 1444
158 | [2]  + 1444 suspended (tty output)  more BackgroundProcsses.md
159 | ```
160 | 
161 | in our shell we just launch more, which happily reads `STDIN` as if
162 | it were a foreground process. We should probably only hook it up if it's
163 | a foreground process, so that background processes don't read `STDIN`
164 | 
165 | ### "Hookup STDIN"
166 | ```go
167 | // If there was an Stdin specified, use it.
168 | if c.Stdin != "" {
169 | 	// Open the file to convert it to an io.Reader
170 | 	if f, err := os.Open(c.Stdin); err == nil {
171 | 		newCmd.Stdin = f
172 | 		defer f.Close()
173 | 	}
174 | } else {
175 | 	// There was no Stdin specified, so 
176 | 	// connect it to the previous process in the
177 | 	// pipeline if there is one, the first process
178 | 	// still uses os.Stdin
179 | 	if i > 0 {
180 | 		pipe, err := cmds[i-1].StdoutPipe()
181 | 		if err != nil {
182 | 			continue
183 | 		}
184 | 		newCmd.Stdin = pipe
185 | 	} else {
186 | 		if !backgroundProcess {
187 | 			newCmd.Stdin = os.Stdin
188 | 		}
189 | 	}
190 | }
191 | ```
192 | 
193 | Hm.. that didn't work. What's going on? If we look into it a little
194 | more into how UNIX handles job control, according to [wikipedia](https://en.wikipedia.org/wiki/Job_control_(Unix)#Implementation)
195 | 
196 | > A background process that attempts to read from or write to its controlling 
197 | > terminal is sent a SIGTTIN (for input) or SIGTTOU (for output) signal. These 
198 | > signals stop the process by default, but they may also be handled in other 
199 | > ways. Shells often override the default stop action of SIGTTOU so that
200 | > background processes deliver their output to the controlling terminal by
201 | > default.
202 | 
203 | So it looks like we'll have to get into signals sooner than we wanted.
204 | `os.Process.Signal()` allows us to send arbitrary signals to programs. So
205 | instead, let's create an io.Reader that, when read from, sends a signal to
206 | a process. In fact, we'll make it a ReadWriter and allow different signals
207 | for read and write, in case we want to implement the `SIGTTOU` behaviour too,
208 | though for now we'll just do `SIGTTIN`.
209 | 
210 | 
211 | ### "Process Signaller"
212 | ```go
213 | type ProcessSignaller struct{
214 | 	// The process to signal when Read from
215 | 	Proc *os.Process
216 | 	ReadSignal, WriteSignal os.Signal
217 | }
218 | 
219 | func (p ProcessSignaller) Read([]byte) (n int, err error) {
220 | 	if p.Proc == nil {
221 | 		return 0, fmt.Errorf("Invalid process.")
222 | 	}
223 | 	if err := p.Proc.Signal(p.ReadSignal); err != nil {
224 | 		return 0, err
225 | 	}
226 | 	return 0, fmt.Errorf("Not an interactive terminal.")
227 | }
228 | 
229 | func (p ProcessSignaller) Write([]byte) (n int, err error) {
230 | 	if p.Proc == nil {
231 | 		return 0, fmt.Errorf("Invalid process.")
232 | 	}
233 | 	if err := p.Proc.Signal(p.WriteSignal); err != nil {
234 | 		return 0, err
235 | 	}
236 | 	return 0, fmt.Errorf("Not an interactive terminal.")
237 | }
238 | ```
239 | 
240 | For now, we'll add this to main.go, though it might eventually deserve
241 | its own file.
242 | 
243 | ### main.go +=
244 | ```go
245 | <<<Process Signaller>>>
246 | ```
247 | 
248 | Now, we'll create the io.ReadWriter while checking if it's a background
249 | process, and hook stdin up to that instead.
250 | 
251 | ### "Handle background inputs"
252 | ```go
253 | var backgroundProcess bool
254 | var stdin io.Reader
255 | if parsed[len(parsed)-1] == "&" {
256 | 	parsed = parsed[:len(parsed)-1]
257 | 	backgroundProcess = true
258 | 	stdin = &ProcessSignaller{
259 | 		// ????
260 | 	}
261 | } else {
262 | 	stdin = os.Stdin
263 | }
264 | ```
265 | 
266 | The `????` is because we don't yet have the `os.Process`, so how can
267 | we instantiate the `ProcessSignaller`?. We'll have to stick with
268 | our original handle background inputs.
269 | 
270 | ### "Handle background inputs"
271 | ```go
272 | var backgroundProcess bool
273 | if parsed[len(parsed)-1] == "&" {
274 | 	// Strip off the &, it's not part of the command.
275 | 	parsed = parsed[:len(parsed)-1]
276 | 	backgroundProcess = true
277 | }
278 | ```
279 | 
280 | Then wait until we have the `exec.Cmd` and are hooking up the `STDIN`
281 | to instantiate our `ProcessSignaller`.
282 | 
283 | In fact, it's worse than that, we *also* don't have the `os.Process`
284 | populated in the `exec.Cmd` instance while hooking up STDIN, because
285 | Cmd.Process isn't populated until the process is started.
286 | 
287 | have two options: change our ProcessSignaller to take an `exec.Cmd`
288 | instead of an `os.Process`, or add a special case after starting.
289 | 
290 | Since the only command hooked up to `STDIN` is the first one, it's
291 | probably easier to do the latter. 
292 | 
293 | ### "Start Processes and Wait"
294 | ```go
295 | for i, c := range cmds {
296 | 	c.Start()
297 | 	if i == 0 && backgroundProcess {
298 | 		c.Stdin = &ProcessSignaller{
299 | 			c.Process,
300 | 			syscall.SIGTTIN,
301 | 			syscall.SIGTTOU,
302 | 		}
303 | 	}
304 | }
305 | if backgroundProcess {
306 | 	// We can't tell if a background process returns an error
307 | 	// or not, so we just claim it didn't.
308 | 	return nil
309 | }
310 | return cmds[len(cmds)-1].Wait()
311 | ```
312 | 
313 | ### "main.go imports" +=
314 | ```go
315 | "syscall"
316 | ```
317 | 
318 | Is this going to work? Can we change `c.Stdin` *after* the process has started?
319 | It compiles and runs, but it turns out the answer is "no", because the behaviour
320 | is the same as before. Let's make our ProcessSignaller a little smarter, then.
321 | Let's have it include a `IsBackground` flag which will cause it to send the
322 | signal, and if `IsBackground` is false, our reader will relay to `os.Stdin`.
323 | 
324 | This should also make it easier to handle `SIGTSTP` (ie. the user pressing 
325 | `^Z` to background a running process) later.
326 | 
327 | ### "Process Signaller"
328 | ```go
329 | type ProcessSignaller struct{
330 | 	// The process to signal when Read from
331 | 	Proc *os.Process
332 | 	ReadSignal, WriteSignal os.Signal
333 | 	IsBackground bool
334 | }
335 | 
336 | func (p ProcessSignaller) Read(b []byte) (n int, err error) {
337 | 	if p.IsBackground == false {
338 | 		return os.Stdin.Read(b)
339 | 	}
340 | 	if p.Proc == nil {
341 | 		return 0, fmt.Errorf("Invalid process.")
342 | 	}
343 | 	if err := p.Proc.Signal(p.ReadSignal); err != nil {
344 | 		return 0, err
345 | 	}
346 | 	return 0, fmt.Errorf("Not an interactive terminal.")
347 | }
348 | 
349 | func (p ProcessSignaller) Write(b []byte) (n int, err error) {
350 | 	if p.IsBackground == false {
351 | 		return os.Stdout.Write(b)
352 | 	}
353 | 
354 | 	if p.Proc == nil {
355 | 		return 0, fmt.Errorf("Invalid process.")
356 | 	}
357 | 	if err := p.Proc.Signal(p.WriteSignal); err != nil {
358 | 		return 0, err
359 | 	}
360 | 	return 0, fmt.Errorf("Not an interactive terminal.")
361 | }
362 | ```
363 | 
364 | Then we always hook up our new reader to STDIN.
365 | 
366 | ### "Hookup STDIN"
367 | ```go
368 | // If there was an Stdin specified, use it.
369 | if c.Stdin != "" {
370 | 	// Open the file to convert it to an io.Reader
371 | 	if f, err := os.Open(c.Stdin); err == nil {
372 | 		newCmd.Stdin = f
373 | 		defer f.Close()
374 | 	}
375 | } else {
376 | 	// There was no Stdin specified, so 
377 | 	// connect it to the previous process in the
378 | 	// pipeline if there is one, the first process
379 | 	// still uses os.Stdin
380 | 	if i > 0 {
381 | 		pipe, err := cmds[i-1].StdoutPipe()
382 | 		if err != nil {
383 | 			continue
384 | 		}
385 | 		newCmd.Stdin = pipe
386 | 	} else {
387 |  		newCmd.Stdin = &ProcessSignaller{
388 | 			newCmd.Process,
389 | 			syscall.SIGTTIN,
390 | 			syscall.SIGTTOU,
391 | 			backgroundProcess,
392 | 		}
393 | 	}
394 | }
395 | ```
396 | 
397 | But then after starting the process, we'll have to set the *os.Process
398 | pointer if Stdin is a ProcessSignaller. (I suppose we could have done this
399 | in the first place too, but I didn't think of it until now.)
400 | 
401 | ### "Start Processes and Wait"
402 | ```go
403 | for _, c := range cmds {
404 | 	c.Start()
405 | 	if ps, ok := c.Stdin.(*ProcessSignaller); ok {
406 | 		ps.Proc = c.Process
407 | 	}
408 | }
409 | if backgroundProcess {
410 | 	// We can't tell if a background process returns an error
411 | 	// or not, so we just claim it didn't.
412 | 	return nil
413 | }
414 | return cmds[len(cmds)-1].Wait()
415 | ```
416 | 
417 | That's.. sort of working, except that user input now seems to be extra slow
418 | when running in the foregroundand there's no indication from more about it
419 |  running in the background or way to resume it.
420 | 
421 | At least it's easy to print an error message before sending a signal.
422 | 
423 | ### "Process Signaller"
424 | ```go
425 | type ProcessSignaller struct{
426 | 	// The process to signal when Read from
427 | 	Proc *os.Process
428 | 	ReadSignal, WriteSignal os.Signal
429 | 	IsBackground bool
430 | }
431 | 
432 | func (p *ProcessSignaller) Read(b []byte) (n int, err error) {
433 | 	if !p.IsBackground {
434 | 		return os.Stdin.Read(b)
435 | 	}
436 | 	if p.Proc == nil {
437 | 		return 0, fmt.Errorf("Invalid process.")
438 | 	}
439 | 	fmt.Fprintf(os.Stderr, "%d suspended (tty input from background)\n", p.Proc.Pid)
440 | 	if err := p.Proc.Signal(p.ReadSignal); err != nil {
441 | 		return 0, err
442 | 	}
443 | 	return 0, fmt.Errorf("Not an interactive terminal.")
444 | }
445 | 
446 | func (p *ProcessSignaller) Write(b []byte) (n int, err error) {
447 | 	if !p.IsBackground {
448 | 		return os.Stdout.Write(b)
449 | 	}
450 | 
451 | 	if p.Proc == nil {
452 | 		return 0, fmt.Errorf("Invalid process.")
453 | 	}
454 | 	fmt.Fprintf(os.Stderr, "%d suspended (tty output from background)\n", p.Proc.Pid)
455 | 	if err := p.Proc.Signal(p.WriteSignal); err != nil {
456 | 		return 0, err
457 | 	}
458 | 	return 0, fmt.Errorf("Not an interactive terminal.")
459 | }
460 | ```
461 | 
462 | Now, what about that speed issue? In fact, it's blocking in the Read() call,
463 | while os.Stdin didn't when set to the processes's stdin directly. I'm not really
464 | certain why that's the case, but we can check if there's data available to return
465 | and send an error instead of blocking in Read when there's no data available.
466 | 
467 | To do that, we'll have to make the terminal that was initialized available
468 | in main() available globally.
469 | 
470 | 
471 | ### "main.go globals" +=
472 | ```go
473 | var terminal *term.Term
474 | ```
475 | 
476 | ### "Initialize Terminal" +=
477 | ```go
478 | terminal = t
479 | ```
480 | 
481 | And then use that to add a check before calling os.Stdin.Read(). We'll send an
482 | io.EOF error along with it, because in non-Go languages the definition of "EOF"
483 | is "a read that returns 0 bytes" in C (where functions can only return 1 value)
484 | so that's how the program that was invoked will be interpreting the 0 byte read
485 | anyways.
486 | 
487 | ### "main.go imports" +=
488 | ```go
489 | "io"
490 | ```
491 | ### "Process Signaller"
492 | ```go
493 | type ProcessSignaller struct{
494 | 	// The process to signal when Read from
495 | 	Proc *os.Process
496 | 	ReadSignal, WriteSignal os.Signal
497 | 	IsBackground bool
498 | }
499 | 
500 | func (p *ProcessSignaller) Read(b []byte) (n int, err error) {
501 | 	if !p.IsBackground {
502 | 		// If there's no data available from os.Stdin,
503 | 		// don't block.
504 | 		if n, err := terminal.Available(); n <= 0 {
505 | 			if err != nil {
506 | 				return n, err
507 | 			}
508 | 			return n, io.EOF
509 | 		}
510 | 		return os.Stdin.Read(b)
511 | 	}
512 | 	if p.Proc == nil {
513 | 		return 0, fmt.Errorf("Invalid process.")
514 | 	}
515 | 	fmt.Fprintf(os.Stderr, "%d suspended (tty input from background)\n", p.Proc.Pid)
516 | 	if err := p.Proc.Signal(p.ReadSignal); err != nil {
517 | 		return 0, err
518 | 	}
519 | 	return 0, fmt.Errorf("Not an interactive terminal.")
520 | }
521 | 
522 | func (p *ProcessSignaller) Write(b []byte) (n int, err error) {
523 | 	if !p.IsBackground {
524 | 		return os.Stdout.Write(b)
525 | 	}
526 | 
527 | 	if p.Proc == nil {
528 | 		return 0, fmt.Errorf("Invalid process.")
529 | 	}
530 | 	fmt.Fprintf(os.Stderr, "%d suspended (tty output from background)\n", p.Proc.Pid)
531 | 	if err := p.Proc.Signal(p.WriteSignal); err != nil {
532 | 		return 0, err
533 | 	}
534 | 	return 0, fmt.Errorf("Not an interactive terminal.")
535 | }
536 | ```
537 | 
538 | We can now *start* a process in the background, without any significant
539 | performance loss, but have no way of sending existing processes to the background,
540 | or resuming them. This exploration of adding a feature to the shell has
541 | already lasted long enough, so let's leave job control for another day.
542 | 


--------------------------------------------------------------------------------
/BackgroundProcessesRevisited.md:
--------------------------------------------------------------------------------
  1 | # Background Processes, Revisited
  2 | 
  3 | We hacked together a method of running processes in the background
  4 | with BackgroundProcesses.md, but unfortunately it isn't a very good
  5 | hack and we should probably do it right.
  6 | 
  7 | In reality, there's no need for the Process Signaller. The OS keeps
  8 | track of what the foreground process is, and sends signals the interrupt
  9 | signals as appropriate.
 10 | 
 11 | All we should be doing is using system calls to tell it when the foreground
 12 | process has switched. So let's start by getting rid of the Process Signaller,
 13 | and hooking up the first process to os.Stdin (unless redirected.) That way
 14 | we avoid the overhead that was causing our shell to be noticablely slow when
 15 | using our custom io.Reader for STDIN
 16 | 
 17 | ### "Process Signaller"
 18 | ```go
 19 | ```
 20 | 
 21 | ### "Hookup STDIN"
 22 | ```go
 23 | // If there was an Stdin specified, use it.
 24 | if c.Stdin != "" {
 25 | 	// Open the file to convert it to an io.Reader
 26 | 	if f, err := os.Open(c.Stdin); err == nil {
 27 | 		newCmd.Stdin = f
 28 | 		defer f.Close()
 29 | 	}
 30 | } else {
 31 | 	// There was no Stdin specified, so 
 32 | 	// connect it to the previous process in the
 33 | 	// pipeline if there is one, the first process
 34 | 	// still uses os.Stdin
 35 | 	if i > 0 {
 36 | 		pipe, err := cmds[i-1].StdoutPipe()
 37 | 		if err != nil {
 38 | 			continue
 39 | 		}
 40 | 		newCmd.Stdin = pipe
 41 | 	} else {
 42 | 		newCmd.Stdin = os.Stdin
 43 | 	}
 44 | }
 45 | ```
 46 | 
 47 | Now, when started a process in Go, exec can take a syscall.*SysProcAttr to
 48 | set system properties of the process that gets created. SysProcAttr has
 49 | a Foreground attribute, but if we used that, we wouldn't be able to create
 50 | our pipeline. As soon as the first process in the pipeline was created, we'd
 51 | be relegated to a background process ourselves.
 52 | 
 53 | What we need to do instead is set the process group ID (PGid) for each of our
 54 | processes, and at the end have the kernel switch that group to the foreground
 55 | process with a syscall.
 56 | 
 57 | syscall.SysProcAttr let's us set the pgid with a Setpgid flag. If we explicitly
 58 | set a Pgid it'll set it to that, otherwise it'll create a new one. We'll let
 59 | the first Pgid be set automatically, and then after starting the first process
 60 | get the Pgid for that process and set the rest of the processes in that group
 61 | to the new pgid.
 62 | 
 63 | We'll also want to keep a list of what process groups exist.
 64 | 
 65 | While we're at it, we should probably want to keep track of what the foreground
 66 | process is. We'll also need to keep track of the pgrp we're creating so that
 67 | we can set that to the foreground process.
 68 | 
 69 | # "main.go globals" +=
 70 | ```go
 71 | var processGroups []uint32
 72 | 
 73 | var ForegroundPid uint32
 74 | ```
 75 | 
 76 | ### "Start Processes and Wait"
 77 | ```go
 78 | <<<Create SysProcAttr with appropriate attributes>>>
 79 | <<<Start processes with proper Pgid>>>
 80 | <<<Change foreground process>>>
 81 | ```
 82 | 
 83 | Let's try the simplest SysProcAttr:
 84 | 
 85 | ### "Create SysProcAttr with appropriate attributes"
 86 | ```
 87 | var pgrp uint32
 88 | sysProcAttr := &syscall.SysProcAttr{
 89 | 	Setpgid: true,
 90 | }
 91 | ```
 92 | 
 93 | And we'll range through cmds. After the first process is created,
 94 | we'll get the Pgid of that process to explicitly set it for further
 95 | processes in the pipeline.
 96 | 
 97 | ### "Start processes with proper Pgid"
 98 | ```go
 99 | for _, c := range cmds {
100 | 	c.SysProcAttr = sysProcAttr
101 | 	if err := c.Start(); err != nil {
102 | 		return err
103 | 	}
104 | 	if sysProcAttr.Pgid == 0 {
105 | 		sysProcAttr.Pgid, _ = syscall.Getpgid(c.Process.Pid)
106 | 		pgrp = uint32(sysProcAttr.Pgid)
107 | 		processGroups = append(processGroups, uint32(c.Process.Pid))
108 | 	}
109 | }
110 | ```
111 | 
112 | Now, how do we handle changing the process group? If we dig into how Go
113 | does it in the test framework, we'll see that is uses a raw syscall to `syscall.SYS_IOCTL`.
114 | It'd be nice if there was something higher level (or if standard syscalls had
115 | better names than `TIOCSPGRP`, but as far as I can tell there isn't.
116 | 
117 | So let's just return if this was a background process otherwise, and otherwise
118 | make the syscall to set the foreground process to the process group we just created.
119 | We should also set our global ForegroundPid variable while we're at it.
120 | 
121 | ### "Change foreground process"
122 | ```go
123 | <<<Return if backgroundProcess>>>
124 | <<<Set ForegroundPid to pgrp>>>
125 | <<<Set Foreground Process to pgrp>>>
126 | ```
127 | 
128 | ### "Return if backgroundProcess"
129 | ```go
130 | if backgroundProcess {
131 | 	// We can't tell if a background process returns an error
132 | 	// or not, so we just claim it didn't.
133 | 	return nil
134 | }
135 | ```
136 | 
137 | ### "Set ForegroundPid to pgrp"
138 | ```go
139 | ForegroundPid = pgrp
140 | ```
141 | 
142 | ### "Set Foreground Process to pgrp"
143 | ```go
144 | _, _, err1 := syscall.RawSyscall(
145 | 	syscall.SYS_IOCTL,
146 | 	uintptr(0),
147 | 	uintptr(syscall.TIOCSPGRP),
148 | 	uintptr(unsafe.Pointer(&pgrp)),
149 | )
150 | // RawSyscall returns an int for the error, we need to compare
151 | // to syscall.Errno(0) instead of nil
152 | if err1 != syscall.Errno(0) {
153 | 	return err1
154 | }
155 | return ForegroundProcess
156 | ```
157 | 
158 | Note that we returned a ForegroundProcess error instead of a nil after setting
159 | the foreground process group. This is to indicate to the caller that the process
160 | spawned was a foreground process, so it should block. (Being a background process
161 | doesn't mean we stop running, it just means any I/O will generate a signal telling
162 | us to stop.)
163 | 
164 | We should probably declare the sentinal error that we're using, too.
165 | 
166 | # "main.go globals" +=
167 | ```go
168 | var ForegroundProcess error = errors.New("Process is a foreground process")
169 | ```
170 | 
171 | One problem with this is that the process we're invoking likely expects the
172 | terminal to be in the normal line mode, not CBreak mode, so we should probably
173 | restore the tty to its original state before changing the active process, and
174 | then set it to CBreak mode again once we get control back. (Ideally, when we
175 | got control we'd check the status of the the terminal and keep it associated
176 | with the process group in case the process was background/foregrounded, but for
177 | now we'll just make the flawed assumption that we're the only one who will
178 | ever change the tty mode, since the term package we're using doesn't seem
179 | to have an easy way to get the current mode.)
180 | 
181 | ### "Set Foreground Process to pgrp"
182 | ```go
183 | terminal.Restore()
184 | _, _, err1 := syscall.RawSyscall(
185 | 	syscall.SYS_IOCTL,
186 | 	uintptr(0),
187 | 	uintptr(syscall.TIOCSPGRP),
188 | 	uintptr(unsafe.Pointer(&pgrp)),
189 | )
190 | // RawSyscall returns an int for the error, we need to compare
191 | // to syscall.Errno(0) instead of nil
192 | if err1 != syscall.Errno(0) {
193 | 	return err1
194 | }
195 | return ForegroundProcess
196 | ```
197 | 
198 | Now, another problem is that once we launch a foreground process, we don't magically
199 | become the foreground process again once it exits. We need to do that ourselves once
200 | a foreground process terminates. But how do we know a foreground process terminated?
201 | 
202 | Whenever a child process dies or changes foreground state, the parent (that's us!)
203 | gets a SIGCHLD signal from unix. The default behaviour for SIGCHLD in Go is to ignore it.
204 | Instead, we can listen for it with the `os/signal` package. `os/signal` will send us
205 | a message on a channel that we specify, and we'll have to wait for something to come in
206 | on that channel if the process launched was a foreground process (that's where our
207 | sentinal error comes in.)
208 | 
209 | We'll create the channel and ask to be notified when we're initializing the terminal.
210 | 
211 | ## "Initialize Terminal" +=
212 | ```go
213 | <<<Create SIGCHLD chan>>>
214 | ```
215 | 
216 | ## "Create SIGCHLD chan"
217 | ```go
218 | child := make(chan os.Signal)
219 | signal.Notify(child, syscall.SIGCHLD)
220 | ```
221 | 
222 | Since we're importing `os/signal`, it may be a good idea to ignore some signals, like
223 | the SIGTTOU we get if we try and print something while not the foreground process.
224 | We probably want to ignore SIGINT (the user pressed ctrl-C) while we're at it.
225 | 
226 | ## "Initialize Terminal" +=
227 | ```go
228 | <<<Ignored certain signal types>>>
229 | ```
230 | 
231 | ## "Ignore certain signal types"
232 | ```go
233 | signal.Ignore(
234 | 	<<<Ignored signal types>>>
235 | )
236 | ```
237 | 
238 | ### "Ignored signal types"
239 | ```go
240 | syscall.SIGTTOU,
241 | syscall.SIGINT,
242 | ```
243 | 
244 | (We've used a new few imports that we should probably add, too:
245 | 
246 | # "main.go imports" +=
247 | ```go
248 | "unsafe"
249 | "errors"
250 | "os/signal"
251 | ```
252 | )
253 | 
254 | Now, our command handler needs to wait for the foreground process status to change if
255 | we returned our ForegroundProcess sentinal.
256 | 
257 | ### "Handle Command"
258 | ```go
259 | if cmd == "exit" || cmd == "quit" {
260 | 	t.Restore()
261 | 	os.Exit(0)
262 | } else if cmd == "" {
263 | 	PrintPrompt()
264 | } else {
265 | 	err := cmd.HandleCmd()
266 | 	if err == ForegroundProcess {
267 | 		Wait(child)
268 | 	} else if err != nil {
269 | 		fmt.Fprintf(os.Stderr, "%v\n", err)
270 | 	}
271 | 	PrintPrompt()
272 | }
273 | ```
274 | 
275 | Let's start by defining the Wait function that we just used:
276 | 
277 | ### "main.go funcs" +=
278 | ```go
279 | func Wait(ch chan os.Signal) {
280 | 	<<<Wait Implementation>>>
281 | }
282 | ```
283 | 
284 | How are we going to implement wait? Well, we'll have to start with waiting
285 | for something to come in on our channel.
286 | 
287 | ### "Wait Channel Select"
288 | ```go
289 | select {
290 | case <-ch:
291 | 	<<<SIGCHLD Received handler>>>
292 | }
293 | ```
294 | 
295 | And then what? All we know is that we got a signal that a child status changed.
296 | We don't even know who sent it. For all we know, a background process died or
297 | finished. So let's just go through all of our processGroups and update our
298 | processGroup list, resuming the foreground status of the shell as appropriate.
299 | 
300 | In fact, since we're deleting from processGroups, let's just make a new slice
301 | and only add the still valid processes, since Go doesn't make it very easy to
302 | delete from a slice while iterating through it.
303 | 
304 | ### "SIGCHLD Received handler"
305 | ```go
306 | newPg := make([]uint32, 0, len(processGroups))
307 | for _, pg := range processGroups {
308 | 	<<<SIGCHLD update processGroup status>>>
309 | }
310 | processGroups = newPg
311 | ```
312 | 
313 | We also want to make sure we don't return from Wait if something else caused
314 | a SIGCHLD, so let's wrap it in an infinite loop and only return when appropriate.
315 | 
316 | ### "Wait Implementation"
317 | ```go
318 | for {
319 | 	<<<Wait Channel Select>>>
320 | 
321 | 	if ForegroundPid == 0 {
322 | 		return
323 | 	}
324 | }
325 | ```
326 | 
327 | 
328 | We want our update status handler to be something like:
329 | 
330 | ### "SIGCHLD update processGroup status"
331 | ```go
332 | <<<SIGCHLD Get pg status>>>
333 | switch {
334 | <<<SIGCHLD Handle pg statuses>>>
335 | }
336 | ```
337 | 
338 | Okay, now how do we get the exited status of the process before it's done? `syscall.Wait4`
339 | returns a `syscall.WaitStatus`, but not until the child exited, and we're in a loop, so we
340 | don't want to block on every process group that we're trying to get the status of.
341 | 
342 | Luckily, there's some options. `WNOHANG` will cause it to not block but just return the status,
343 | which is exactly what we want. (`WUNTRACED` and `WCONTINUED` just affect which information is
344 | available from the WaitStatus.)
345 | 
346 | ### "SIGCHLD Get pg status"
347 | ```go
348 | var status syscall.WaitStatus
349 | pid1, err := syscall.Wait4(int(pg), &status, syscall.WNOHANG|syscall.WUNTRACED|syscall.WCONTINUED, nil)
350 | if pid1 == 0 && err == nil {
351 | 	// We don't want to accidentally remove things from processGroups if there was an error
352 | 	// from wait.
353 | 	newPg = append(newPg, pg)
354 | 	continue
355 | }
356 | ```
357 | 
358 | We'll add cases for each of the booleans that WaitStatus provides methods to extract.
359 | 
360 | ### "SIGCHLD Handle pg statuses"
361 | ```go
362 | case status.Continued(): 
363 | 	<<<SIGCHLD Handle Continued>>>
364 | case status.Stopped():
365 | 	<<<SIGCHLD Handle Stopped>>>
366 | case status.Signaled():
367 | 	<<<SIGCHLD Handle Signaled>>>
368 | case status.Exited():
369 | 	<<<SIGCHLD Handle Exited>>>
370 | default:
371 | 	<<<SIGCHLD Default Handler>>>
372 | ```
373 | 
374 | Going through them one by one:
375 | 
376 | If the process is resuming because of a SIGCONT, we'll keep the process in our newPg list (it's
377 | still alive), and make it the foreground process if there's no other foreground process.
378 | ### "SIGCHLD Handle Continued"
379 | ```go
380 | newPg = append(newPg, pg)
381 | 
382 | if ForegroundPid == 0 {
383 | 	<<<Make pg foreground>>>
384 | }
385 | ```
386 | 
387 | If the child was stopped and is the foreground process, we should resume our shell as the foreground
388 | process, keep a reference to the process group, and print a message.
389 | 
390 | ### "SIGCHLD Handle Stopped"
391 | ```go
392 | newPg = append(newPg, pg)
393 | if pg == ForegroundPid && ForegroundPid != 0 {
394 | 	<<<Resume Shell Foreground>>>
395 | }
396 | fmt.Fprintf(os.Stderr, "%v is stopped\n", pid1)
397 | ```
398 | 
399 | Signaled means the process died from a signal (as opposed to by calling exit.) In this case we
400 | *don't* want to add it to the newPg list, but *do* want to resume the shell if it was the foreground
401 | process. (We should also tell the user.)
402 | 
403 | ### "SIGCHLD Handle Signaled"
404 | ```go
405 | if pg == ForegroundPid && ForegroundPid != 0 {
406 | 	<<<Resume Shell Foreground>>>
407 | }
408 | 
409 | fmt.Fprintf(os.Stderr, "%v terminated by signal %v\n", pg, status.StopSignal())
410 | ```
411 | 
412 | Exited means it exited normally. If it was the foreground process, we want to resume the shell.
413 | If it was a background process, we want to tell the user. Either way, we should set the `$?`
414 | environment variable, so that it's available from our shell.
415 | 
416 | ### "SIGCHLD Handle Exited"
417 | ```go
418 | if pg == ForegroundPid && ForegroundPid != 0 {
419 | 	<<<Resume Shell Foreground>>>
420 | } else {
421 | 	fmt.Fprintf(os.Stderr, "%v exited (exit status: %v)\n", pid1, status.ExitStatus())
422 | }
423 | os.Setenv("?", strconv.Itoa(status.ExitStatus()))
424 | ```
425 | 
426 | ### "main.go imports" +=
427 | ```go
428 | "strconv"
429 | ```
430 | 
431 | Finally, for the default case, we'll make sure we don't accidentally lose a process from our list
432 | and print a message just so we know that it's happening (even though it probably means we missed
433 | a case statement.) 
434 | 
435 | ### "SIGCHLD Default Handler"
436 | ```go
437 | newPg = append(newPg, pg)
438 | fmt.Fprintf(os.Stderr, "Still running: %v: %v\n", pid1, status)
439 | ```
440 | 
441 | We already worked out the syscall for Make pg foreground and Resume Shell Foreground in our HandleCmd,
442 | but our variable names are different.
443 | 
444 | ### "Make pg foreground"
445 | ```go
446 | terminal.Restore()
447 | var pid uint32 = pg
448 | _, _, err3 := syscall.RawSyscall(
449 | 	syscall.SYS_IOCTL,
450 | 	uintptr(0),
451 | 	uintptr(syscall.TIOCSPGRP),
452 | 	uintptr(unsafe.Pointer(&pid)),
453 | )
454 | if err3 != syscall.Errno(0) {
455 | 	panic(fmt.Sprintf("Err: %v", err3))
456 | }
457 | ForegroundPid = pid
458 | ```
459 | 
460 | Resuming the shell as the foreground is similar, except the process group is our own PID, and we want
461 | to make sure we set the ForegroundPid to 0.
462 | 
463 | ### "Resume Shell Foreground"
464 | ```go
465 | terminal.SetCbreak()
466 | var mypid uint32 = uint32(syscall.Getpid())
467 | _, _, err3 := syscall.RawSyscall(
468 | 	syscall.SYS_IOCTL,
469 | 	uintptr(0),
470 | 	uintptr(syscall.TIOCSPGRP),
471 | 	uintptr(unsafe.Pointer(&mypid)),
472 | )
473 | if err3 != syscall.Errno(0) {
474 | 	panic(fmt.Sprintf("Err: %v", err3))
475 | }
476 | ForegroundPid = 0
477 | ```
478 | 
479 | ## Builtins
480 | 
481 | We've finally got support for background processes! Almost. Ctrl-Z works, but we don't
482 | have any way to trigger it being resumed.
483 | 
484 | We don't have a way to resume it, even if our code handles the case. We'll add a "jobs"
485 | builtin to print the processGroups, and a bg and fg command to send them a signal to
486 | stop or resume.
487 | 
488 | ### "Builtin Commands" +=
489 | ```go
490 | case "jobs":
491 | 	<<<Handle jobs>>>
492 | case "bg":
493 | 	<<<Handle bg>>>
494 | case "fg":
495 | 	<<<Handle fg>>>
496 | 
497 | ```
498 | 
499 | The jobs case is easy:
500 | 
501 | ### "Handle jobs"
502 | ```go
503 | fmt.Printf("Job listing:\n\n")
504 | for i, leader := range processGroups {
505 | 	fmt.Printf("Job %d (%d)\n", i, leader)
506 | }
507 | return nil
508 | ```
509 | 
510 | The bg case shouldn't be very difficult either. We just need to parse
511 | the first argument, convert it to an int, and then get send a `SIGCONT`
512 | signal to the process to tell it to continue.
513 | 
514 | 
515 | ### "Handle bg"
516 | ```go
517 | if len(args) < 1 {
518 | 	return fmt.Errorf("Must specify job to background.")
519 | }
520 | i, err := strconv.Atoi(args[0])
521 | if err != nil {
522 | 	return err
523 | }
524 | 
525 | if i >= len(processGroups) || i < 0 {
526 | 	return fmt.Errorf("Invalid job id %d", i)
527 | }
528 | p, err := os.FindProcess(int(processGroups[i]))
529 | if err != nil {
530 | 	return err
531 | }
532 | if err := p.Signal(syscall.SIGCONT); err != nil {
533 | 	return err
534 | }
535 | return nil
536 | ```
537 | 
538 | fg is similar, except we also want to set ForegroundPid and 
539 | send our TIOCSPGRP syscall to the Pid and return the ForegroundProcess
540 | sentinal.
541 | 
542 | ### "Handle fg"
543 | ```go
544 | if len(args) < 1 {
545 | 	return fmt.Errorf("Must specify job to foreground.")
546 | }
547 | i, err := strconv.Atoi(args[0])
548 | if err != nil {
549 | 	return err
550 | }
551 | 
552 | if i >= len(processGroups) || i < 0 {
553 | 	return fmt.Errorf("Invalid job id %d", i)
554 | }
555 | p, err := os.FindProcess(int(processGroups[i]))
556 | if err != nil {
557 | 	return err
558 | }
559 | if err := p.Signal(syscall.SIGCONT); err != nil {
560 | 	return err
561 | }
562 | terminal.Restore()
563 | var pid uint32 = processGroups[i]
564 | _, _, err3 := syscall.RawSyscall(
565 | 	syscall.SYS_IOCTL,
566 | 	uintptr(0),
567 | 	uintptr(syscall.TIOCSPGRP),
568 | 	uintptr(unsafe.Pointer(&pid)),
569 | )
570 | if err3 != syscall.Errno(0) {
571 | 	panic(fmt.Sprintf("Err: %v", err3))
572 | } else {
573 | 	ForegroundPid = pid
574 | 	return ForegroundProcess
575 | }
576 | ```
577 | 
578 | And *now* we can use our shell for real with job control.
579 | 


--------------------------------------------------------------------------------
/Environment.md:
--------------------------------------------------------------------------------
  1 | # Environment Variables and Startup Scripts
  2 | 
  3 | In a UNIX system, when you execute a progress, the new process inherits
  4 | the environment of the parent who spawned it. This has worked for us well so
  5 | far, because we inherited the `$PATH` of the shell that spawned us, which let
  6 | the `os/exec` package search the path without us doing anything special.
  7 | 
  8 | Unfortunately, it also means we inherited some variables that don't make sense
  9 | any more (like `$SHELL`), and it means that we can't be used as a login shell,
 10 | because there's no way to set environment variables yet.
 11 | 
 12 | We'll implement a `set` builtin, which takes two parameters: the name, and the
 13 | value of an environment variable to set. We don't need a way to launch processes
 14 | with different environments, because the standard Unix command `env` already
 15 | provides that, but we can't depend on an external command for setting an
 16 | environment, because any changes it makes to the environment would only be valid
 17 | for the child process and end once the spawned program dies.
 18 | 
 19 | We also need to provide a way to read and execute startup file (to bootstrap
 20 | the user's environment), which needs to be run in our own process space for much
 21 | the same reason.
 22 | 
 23 | ## Expanding on builtins
 24 | 
 25 | Recall that our most recent HandleCmd implementation was:
 26 | 
 27 | ### "HandleCmd Implementation"
 28 | ```go
 29 | func (c Command) HandleCmd() error {
 30 | 	parsed := c.Tokenize()
 31 | 	<<<Handle no tokens in command case>>>
 32 | 	<<<Replace environment variables in command>>>
 33 | 	<<<Handle background inputs>>>
 34 | 	<<<Handle cd command>>>
 35 | 	<<<Execute command and return>>>
 36 | }
 37 | ```
 38 | 
 39 | Let's change that "Handle cd" to be a more generic "Handle builtins", because
 40 | we'll probably want to be adding more builtins eventually.
 41 | 
 42 | ### "HandleCmd Implementation"
 43 | ```go
 44 | func (c Command) HandleCmd() error {
 45 | 	parsed := c.Tokenize()
 46 | 	<<<Handle no tokens in command case>>>
 47 | 	<<<Replace environment variables in command>>>
 48 | 	<<<Handle background inputs>>>
 49 | 	<<<Handle builtin commands>>>
 50 | 	<<<Execute command and return>>>
 51 | }
 52 | ```
 53 | 
 54 | The builtins can be handled with a fairly simple switch (we'll separate the
 55 | macros into a different macro so that we can easily add to it when we have
 56 | more builtins.)
 57 | 
 58 | ### "Handle builtin commands"
 59 | ```go
 60 | switch parsed[0] {
 61 | 	<<<Builtin Commands>>>
 62 | }
 63 | ```
 64 | 
 65 | ### "Builtin Commands"
 66 | ```go
 67 | case "cd":
 68 | 	<<<Handle cd command>>>
 69 | ```
 70 | 
 71 | The extra unnecessary if statement in Handle cd is going to drive me insane now
 72 | that it's a switch, so let's get rid of it
 73 | 
 74 | ### "Handle cd command"
 75 | ```go
 76 | if len(args) == 0 {
 77 | 	return fmt.Errorf("Must provide an argument to cd")
 78 | }
 79 | return os.Chdir(args[0])
 80 | ```
 81 | 
 82 | And we can now add our "set" builtin.
 83 | 
 84 | ### "Builtin Commands" +=
 85 | ```go
 86 | case "set":
 87 | if len(args) != 2 {
 88 | 	return fmt.Errorf("Usage: set var value")
 89 | }
 90 | return os.Setenv(args[0], args[1])
 91 | ```
 92 | 
 93 | That was pretty straight forward. Now, let's also set the `$SHELL`
 94 | variable on startup and source a startup script. We'll add it to
 95 | the mainbody that we defined at the start of this shell, after
 96 | initializing the terminal but before going into the command loop.
 97 | 
 98 | ### "mainbody"
 99 | ```go
100 | <<<Initialize Terminal>>>
101 | <<<Initialize Shell>>>
102 | <<<Command Loop>>>
103 | ```
104 | 
105 | ### "Initialize Shell"
106 | ```go
107 | os.Setenv("SHELL", os.Args[0])
108 | <<<Read startup script>>>
109 | ```
110 | 
111 | Before reading the startup script, maybe it would make sense to add a
112 | "source" builtin, since the code if effectively the same. We'll define
113 | the function:
114 | 
115 | ### "main.go funcs" +=
116 | ```go
117 | func SourceFile(filename string) error {
118 | 	<<<SourceFile implementation>>>
119 | }
120 | ```
121 | 
122 | and then we just need to call it with the startup script name. What
123 | is the script name? We'll call it `$HOME/.goshrc`, and we'll use the
124 | `os/user` package to look up the `$HOME` directory, just in case there's
125 | any OS specific idiosyncrasies.
126 | 
127 | ### "main.go imports" +=
128 | ```go
129 | "os/user"
130 | ```
131 | 
132 | ### "Read startup script"
133 | ```go
134 | if u, err := user.Current(); err == nil {
135 | 	SourceFile(u.HomeDir + "/.goshrc")
136 | }
137 | ```
138 | 
139 | Let's define the builtin, too. We'll call the builtin `source`, and might
140 | as well just make it source all of the arguments
141 | 
142 | ### "Builtin Commands" +=
143 | ```go
144 | case "source":
145 | 	<<<Source Builtin>>>
146 | ```
147 | 
148 | ### "Source Builtin" 
149 | ```go
150 | if len(args) < 1 {
151 | 	return fmt.Errorf("Usage: source file [...other files]")
152 | }
153 | 
154 | for _, f := range args {
155 | 	SourceFile(f)
156 | }
157 | return nil
158 | ```
159 | 
160 | Okay, now how do we actually implement SourceFile?
161 | 
162 | We'll obviously need to start by opening the file, then we need
163 | to go through it line by line, and then we'll need to execute
164 | each line as if the user had typed it.
165 | then 
166 | ### "SourceFile implementation"
167 | ```go
168 | <<<Open sourced file>>>
169 | <<<Iterate through sourced file>>>
170 | ```
171 | 
172 | ### "Open sourced file"
173 | ```go
174 | f, err := os.Open(filename)
175 | if err != nil {
176 | 	return err
177 | }
178 | defer f.Close()
179 | ```
180 | 
181 | We can iterate through the file by using a bufio.Reader and
182 | reading until there's a '\n'.
183 | 
184 | ### "Iterate through sourced file"
185 | ```go
186 | scanner := bufio.NewReader(f)
187 | for {
188 | 	line, err := scanner.ReadString('\n')
189 | 	switch err {
190 | 		case io.EOF:
191 | 			return nil
192 | 		case nil:
193 | 			// Nothing special
194 | 		default:
195 | 			return err	
196 | 	}
197 | 	<<<Handle sourced file line>>>
198 | }
199 | ```
200 | 
201 | Now, we can handle a line by treating it the same way we treat user entered
202 | input: with the `HandleCmd()` method on a `Command`.
203 | 
204 | ### "Handle sourced file line"
205 | ```go
206 | c := Command(line)
207 | if err := c.HandleCmd(); err != nil {
208 | 	return err
209 | }
210 | ```
211 | 


--------------------------------------------------------------------------------
/Globbing.md:
--------------------------------------------------------------------------------
  1 | # File Globbing
  2 | 
  3 | In order to be really useful as a login shell, we need to support file globbing.
  4 | 
  5 | For instance, `ls *.go` should be expanded to `ls [all the files in the directory
  6 | with the extenion .go]` or `ls ~/` should display our home directory. The Go
  7 | standard lib package `path/filepath` has a `Glob` function, we just need to
  8 | decide how to use it (and potentially handle `~`).
  9 | 
 10 | Recall that our last HandleCmd implementation was:
 11 | 
 12 | ### "HandleCmd Implementation"
 13 | ```go
 14 | func (c Command) HandleCmd() error {
 15 | 	parsed := c.Tokenize()
 16 | 	<<<Handle no tokens in command case>>>
 17 | 	<<<Replace environment variables in command>>>
 18 | 	<<<Handle background inputs>>>
 19 | 	<<<Handle builtin commands>>>
 20 | 	<<<Execute command and return>>>
 21 | }
 22 | ```
 23 | 
 24 | We probably want to do it after replacing environment variables, in case the
 25 | environment variables contain globs.
 26 | 
 27 | Let's try.
 28 | 
 29 | ### "HandleCmd Implementation"
 30 | ```go
 31 | func (c Command) HandleCmd() error {
 32 | 	parsed := c.Tokenize()
 33 | 	<<<Handle no tokens in command case>>>
 34 | 	<<<Replace environment variables in command>>>
 35 | 	<<<Expand file glob tokens>>>
 36 | 	<<<Handle background inputs>>>
 37 | 	<<<Handle builtin commands>>>
 38 | 	<<<Execute command and return>>>
 39 | }
 40 | ```
 41 | 
 42 | Let's just create a new parsed and append the expanded tokens to it as we find
 43 | them, then at the end we can set parsed to our new `newparsed` variable in case
 44 | any of the other blocks reference it. (In fact, "parsed" isn't used after this
 45 | point, "args" is, so let's do it for "args")
 46 | 
 47 | ### "Expand file glob tokens"
 48 | ```go
 49 | // newargs will be at least len(parsed in size, so start by allocating a slice
 50 | // of that capacity
 51 | newargs := make([]string, 0, len(args))
 52 | for _, token := range args {
 53 | 	expanded, err := filepath.Glob(token)
 54 | 	if err != nil || len(expanded) == 0 {
 55 | 		newargs = append(newargs, token)
 56 | 		continue
 57 | 	}
 58 | 	newargs = append(newargs, expanded...)
 59 | 
 60 | }
 61 | args = newargs
 62 | ```
 63 | 
 64 | ### "main.go imports" +=
 65 | ```go
 66 | "path/filepath"
 67 | ```
 68 | 
 69 | "*", "?" and "[]" work as expected, but "~" doesn't. We'll have to manually
 70 | check if an argument starts with "~" and expand it. We should probably make
 71 | sure we match `~user/foo` too, so let's use the regexp .
 72 | 
 73 | ### "Expand file glob tokens"
 74 | ```go
 75 | homedirRe := regexp.MustCompile("^~([a-zA-Z]*)?(/)?")
 76 | // newargs will be at least len(parsed in size, so start by allocating a slice
 77 | // of that capacity
 78 | newargs := make([]string, 0, len(args))
 79 | for _, token := range args {
 80 | 	<<<Replace tilde with homedir in token>>>
 81 | 	expanded, err := filepath.Glob(token)
 82 | 	if err != nil || len(expanded) == 0 {
 83 | 		newargs = append(newargs, token)
 84 | 		continue
 85 | 	}
 86 | 	newargs = append(newargs, expanded...)
 87 | 
 88 | }
 89 | args = newargs
 90 | ```
 91 | 
 92 | The fact that regexp needs to be compiled every time we call HandleCmd is
 93 | annoying, so let's move it into a global that only gets compiled on startup
 94 | (and that we can use elsewhere.)
 95 | 
 96 | ### "Expand file glob tokens"
 97 | ```go
 98 | // newargs will be at least len(parsed in size, so start by allocating a slice
 99 | // of that capacity
100 | newargs := make([]string, 0, len(args))
101 | for _, token := range args {
102 | 	<<<Replace tilde with homedir in token>>>
103 | 	expanded, err := filepath.Glob(token)
104 | 	if err != nil || len(expanded) == 0 {
105 | 		newargs = append(newargs, token)
106 | 		continue
107 | 	}
108 | 	newargs = append(newargs, expanded...)
109 | 
110 | }
111 | args = newargs
112 | ```
113 | 
114 | ### "main.go globals" +=
115 | ```go
116 | var homedirRe *regexp.Regexp = regexp.MustCompile("^~([a-zA-Z]*)?(/*)?")
117 | ```
118 | 
119 | ### "Replace tilde with homedir in token"
120 | ```
121 | if match := homedirRe.FindStringSubmatch(token); match != nil {
122 | 	var u *user.User
123 | 	var err error
124 | 	if match[1] != "" {
125 | 		u, err = user.Lookup(match[1])
126 | 	} else {
127 | 		u, err = user.Current()
128 | 	}
129 | 	if err == nil {
130 | 		token = strings.Replace(token, match[0], u.HomeDir + "/", 1)
131 | 	}
132 | }
133 | ```
134 | 
135 | ### "main.go imports" +=
136 | ```go
137 | "os/user"
138 | "strings"
139 | ```
140 | 
141 | We can do ls ~/ or ls ~root now, but we have a problem where tab completion
142 | isn't smart enough to look `~` style directories.
143 | 
144 | Let's move our replacer code into a function so that we don't have to duplicate
145 | it here.
146 | 
147 | ### "main.go funcs" +=
148 | ```go
149 | func replaceTilde(s string) string {
150 | 	<<<replaceTilde implementation>>>
151 | }
152 | ```
153 | 
154 | ### "replaceTilde implementation"
155 | ```go
156 | if match := homedirRe.FindStringSubmatch(s); match != nil {
157 | 	var u *user.User
158 | 	var err error
159 | 	if match[1] != "" {
160 | 		u, err = user.Lookup(match[1])
161 | 	} else {
162 | 		u, err = user.Current()
163 | 	}
164 | 	if err == nil {
165 | 		return strings.Replace(s, match[0], u.HomeDir, 1)
166 | 	}
167 | }
168 | return s
169 | ```
170 | 
171 | Then we just can use while expanding in HandleCmd
172 | 
173 | ### "Replace tilde with homedir in token"
174 | ```go
175 | token = replaceTilde(token)
176 | ```
177 | 
178 | As for file suggestions, we had this:
179 | 
180 | ### "File Suggestions Implementation"
181 | ```go
182 | <<<Check base dir>>>
183 | 
184 | filedir := filepath.Dir(base)
185 | fileprefix := filepath.Base(base)
186 | files, err := ioutil.ReadDir(filedir)
187 | if err != nil {
188 | 	return nil
189 | }
190 | 
191 | <<<Check files for matches and return>>>
192 | ```
193 | 
194 | We should be able to just blindly replace base with the tilde replaced version
195 | at the start of the function.
196 | 
197 | ### "File Suggestions Implementation"
198 | ```go
199 | base = replaceTilde(base)
200 | <<<Check base dir>>>
201 | 
202 | filedir := filepath.Dir(base)
203 | fileprefix := filepath.Base(base)
204 | files, err := ioutil.ReadDir(filedir)
205 | if err != nil {
206 | 	return nil
207 | }
208 | 
209 | <<<Check files for matches and return>>>
210 | ```
211 | 
212 | This results in the "~" being expanded directly on the commandline when we push
213 | tab. This probably isn't a big deal, and might even a good thing since it might
214 | make people stop believing it's a character with special meaning in filenames
215 | outside of the shell.
216 | 
217 | Testing this reveals another problem that we've always had: when we tab complete
218 | a directory, the "/" at the end gets duplicated each time we hit tab. We should
219 | be able to just use `filepath.Clean` before appending the file to our matches.
220 | 
221 | Our code was:
222 | 
223 | ### "Append file match"
224 | ```go
225 | if filedir != "/" {
226 | 	matches = append(matches, filedir + "/" + name)
227 | } else {
228 | 	matches = append(matches, filedir + name)
229 | }
230 | ```
231 | 
232 | And now we might even be able to get rid of the if statement
233 | 
234 | ### "Append file match"
235 | ```go
236 | matches = append(matches, filepath.Clean(filedir + "/" + name))
237 | ```
238 | 
239 | (This also fixes an annoyance where "./" would get prepended to the beginning
240 | of file names when tab completing, and makes tab general clean up file paths.)
241 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | MDFILES=README.md Tokenization.md TabCompletion.md Piping.md \
 2 | 	BackgroundProcesses.md Environment.md BackgroundProcessesRevisited.md \
 3 | 	TabCompletionRevisited.md Globbing.md Prompts.md
 4 | 
 5 | all: $(MDFILES)
 6 | 	lmt $(MDFILES)
 7 | 	go fmt .
 8 | 	go test .
 9 | 	go build .
10 | 
11 | 


--------------------------------------------------------------------------------
/Piping.md:
--------------------------------------------------------------------------------
  1 | # Piping and Redirection
  2 | 
  3 | To be a useful UNIX shell, our shell needs to support piping and
  4 | redirection. Since while we were writing our tokenizer we had the
  5 | foresight of handling `|`, `<`, and `>` characters, we should only
  6 | need to update our HandleCmd implementation.
  7 | 
  8 | Recall, our most recent HandleCmd implementation looked like this:
  9 | 
 10 | ### "HandleCmd Implementation"
 11 | ```go
 12 | func (c Command) HandleCmd() error {
 13 | 	parsed := c.Tokenize()
 14 | 	if len(parsed) == 0 {
 15 | 		// There was no command, it's not an error, the user just hit
 16 | 		// enter.
 17 | 		PrintPrompt()
 18 | 		return nil
 19 | 	}
 20 | 
 21 | 	var args []string
 22 | 	for _, val := range parsed[1:] {
 23 | 		if val[0] == '$' {
 24 | 			args = append(args, os.Getenv(val[1:]))
 25 | 		} else {
 26 | 			args = append(args, val)
 27 | 		}
 28 | 	}
 29 | 
 30 | 	if parsed[0] == "cd" {
 31 | 		if len(args) == 0 {
 32 | 			return fmt.Errorf("Must provide an argument to cd")
 33 | 		}
 34 | 		return os.Chdir(args[0])
 35 | 
 36 | 	}
 37 | 
 38 | 	cmd := exec.Command(parsed[0], args...)
 39 | 	cmd.Stdin = os.Stdin
 40 | 	cmd.Stdout = os.Stdout
 41 | 	cmd.Stderr = os.Stderr
 42 | 
 43 | 	return cmd.Run()
 44 | }
 45 | ```
 46 | 
 47 | Let's refactor the way we markup the sections a little before we go any further
 48 | to make it easier to maintain:
 49 | 
 50 | ### "HandleCmd Implementation"
 51 | ```go
 52 | func (c Command) HandleCmd() error {
 53 | 	parsed := c.Tokenize()
 54 | 	<<<Handle no tokens in command case>>>
 55 | 	<<<Replace environment variables in command>>>
 56 | 	<<<Handle cd command>>>
 57 | 	<<<Execute command and return>>>
 58 | }
 59 | ```
 60 | 
 61 | ### "Handle no tokens in command case"
 62 | ```go
 63 | if len(parsed) == 0 {
 64 | 	// There was no command, it's not an error, the user just hit
 65 | 	// enter.
 66 | 	PrintPrompt()
 67 | 	return nil
 68 | }
 69 | ```
 70 | 
 71 | ### "Replace environment variables in command"
 72 | ```go
 73 | var args []string
 74 | for _, val := range parsed[1:] {
 75 | 	if val[0] == '$' {
 76 | 		args = append(args, os.Getenv(val[1:]))
 77 | 	} else {
 78 | 		args = append(args, val)
 79 | 	}
 80 | }
 81 | ```
 82 | 
 83 | ### "Handle cd command"
 84 | ```go
 85 | if parsed[0] == "cd" {
 86 | 	if len(args) == 0 {
 87 | 		return fmt.Errorf("Must provide an argument to cd")
 88 | 	}
 89 | 	return os.Chdir(args[0])
 90 | }
 91 | ```
 92 | 
 93 | ### "Execute command and return"
 94 | ```go
 95 | cmd := exec.Command(parsed[0], args...)
 96 | cmd.Stdin = os.Stdin
 97 | cmd.Stdout = os.Stdout
 98 | cmd.Stderr = os.Stderr
 99 | 
100 | return cmd.Run()
101 | ```
102 | 
103 | It's only the last part we need to be concerned about to implement pipelining
104 | and redirection. Instead of starting one command, we need to start multiple
105 | commands and hook up their stdin and stdout streams.
106 | 
107 | Let's start by going through the tokens looking for a `|` token. Each time we 
108 | find one, we'll add all the elements since the last `|` to a slice of [][]string,
109 | with each one representing a command in the pipeline. Then we'll go through
110 | those and find the redirections, and parse those out, then we can create
111 | a slice of []*exec.Cmd that we can create with exec.Command() with the name
112 | and args..
113 | 
114 | Actually, this is getting too complicated. Let's create a new type instead.
115 | Something like:
116 | 
117 | ### "Parsed Command Type"
118 | ```go
119 | type ParsedCommand struct{
120 | 	Args []string
121 | 	Stdin string
122 | 	Stdout string
123 | }
124 | ```
125 | ### "main.go globals" +=
126 | ```go
127 | <<<Parsed Command Type>>>
128 | ```
129 | 
130 | Then we can go through a single pass and created the []ParsedCommand, which
131 | will make it easier to create the []*exec.Cmd without needing to make extra
132 | passes to figure out which tokens are part of the command, which are arguments,
133 | and which are special shell characters like redirection and pipe.
134 | 
135 | So our execute command becomes something like:
136 | 
137 | ### "Execute command and return"
138 | ```go
139 | // Keep track of the current command being built
140 | var currentCmd ParsedCommand
141 | // Keep array of all commands that have been built, so we can create the
142 | // pipeline
143 | var allCommands []ParsedCommand
144 | // Keep track of where this command started in parsed, so that we can build
145 | // currentCommand.Args when we find a special token.
146 | var lastCommandStart = 0
147 | // Keep track of if we've found a special token such as < or >, so that
148 | // we know if currentCmd.Args has already been populated.
149 | var foundSpecial bool
150 | for i, t := parsed {
151 | 	if t == "<" || t == ">" || t == "|" {
152 | 		if foundSpecial == false {
153 | 			currrentCmd.Args = parsed[lastCommandStart] 
154 | 		}
155 | 		foundSpecial = true
156 | 	}
157 | 	if t == "|" {
158 | 		allCommands = append(allCommands, currentCmd)
159 | 		lastCommandStart = i
160 | 		foundSpecial = false
161 | 	}
162 | }
163 | 
164 | <<<Build pipeline and execute>>>
165 | ```
166 | 
167 | Our tokens code is probably getting complicated enough that we should have a
168 | Token type with methods like `IsSpecial() bool` and `IsPipe() bool` instead
169 | of a []string to make it more readable, but we'll ignore that for now.
170 | 
171 | I'm not very confident in the above, either, so let's refactor it into a function
172 | and build another table driven test.
173 | 
174 | ### "Execute command and return"
175 | ```go
176 | // Convert parsed from []string to []Token. We should refactor all the code
177 | // to use tokens, but for now just do this instead of going back and changing
178 | // all the references/declarations in every other section of code.
179 | var parsedtokens []Token = []Token{Token(parsed[0])}
180 | for _, t := range args {
181 | 	parsedtokens = append(parsedtokens, Token(t))	
182 | }
183 | commands := ParseCommands(parsedtokens)
184 | <<<Build pipeline and execute>>>
185 | ```
186 | 
187 | ### "tokenize.go globals" +=
188 | ```go
189 | type Token string
190 | 
191 | func (t Token) IsPipe() bool {
192 | 	return t == "|"
193 | }
194 | 
195 | func (t Token) IsSpecial() bool {
196 | 	return t == "<" || t == ">" || t == "|" 
197 | }
198 | 
199 | func (t Token) IsStdinRedirect() bool {
200 | 	return t == "<"
201 | }
202 | 
203 | func (t Token) IsStdoutRedirect() bool {
204 | 	return t == ">"
205 | }
206 | ```
207 | 
208 | ### "main.go funcs" +=
209 | ```go
210 | func ParseCommands(tokens []Token) []ParsedCommand {
211 | 	<<<ParseCommands Implementation>>>
212 | }
213 | ```
214 | 
215 | ### "ParseCommands Implementation"
216 | ```go
217 | // Keep track of the current command being built
218 | var currentCmd ParsedCommand
219 | // Keep array of all commands that have been built, so we can create the
220 | // pipeline
221 | var allCommands []ParsedCommand
222 | // Keep track of where this command started in parsed, so that we can build
223 | // currentCommand.Args when we find a special token.
224 | var lastCommandStart = 0
225 | // Keep track of if we've found a special token such as < or >, so that
226 | // we know if currentCmd.Args has already been populated.
227 | var foundSpecial bool
228 | for i, t := range tokens {
229 | 	if t.IsSpecial() {
230 | 		if foundSpecial == false {
231 | 			// Convert from Token to string
232 | 			var slice []Token
233 | 			if i == len(tokens)-1 {
234 | 				slice = tokens[lastCommandStart:]
235 | 			} else {
236 | 				slice = tokens[lastCommandStart:i]
237 | 			}
238 | 
239 | 			for _, t := range slice {
240 | 				currentCmd.Args = append(currentCmd.Args, string(t))
241 | 			}
242 | 		}
243 | 		foundSpecial = true
244 | 	}
245 | 	if t.IsPipe() {
246 | 		allCommands = append(allCommands, currentCmd)
247 | 		lastCommandStart = i
248 | 		foundSpecial = false
249 | 	}
250 | }
251 | allCommands = append(allCommands, currentCmd)
252 | return allCommands
253 | ```
254 | 
255 | We'll add the tests to the existing tokenize_test, since it's still related
256 | to parsing the command.
257 | 
258 | ### tokenize_test.go +=
259 | ```go
260 | func TestParseCommands(t *testing.T) {
261 | 	tests := []struct{
262 | 		val []Token
263 | 		expected []ParsedCommand
264 | 	}{
265 | 		{
266 | 			[]Token{"ls"},
267 | 			[]ParsedCommand{
268 | 				ParsedCommand{[]string{"ls"}, "", ""},
269 | 			},
270 | 		},
271 | 		{
272 | 			[]Token{"ls", "|", "cat"}, 
273 | 			[]ParsedCommand{
274 | 				ParsedCommand{[]string{"ls"}, "", ""},
275 | 				ParsedCommand{[]string{"cat"}, "", ""},
276 | 			},
277 | 		},
278 | 		<<<Other ParseCommands Test Cases>>>
279 | 	}
280 | 
281 | 	for i, tc := range tests {
282 | 		val := ParseCommands(tc.val)
283 | 		if len(val) != len(tc.expected) {
284 | 			t.Fatalf("Unexpected number of ParsedCommands in test %d. Got %v want %v", i, val, tc.expected)
285 | 		}
286 | 		for j, _ := range val {
287 | 			if val[j].Stdin != tc.expected[j].Stdin {
288 | 				t.Fatalf("Mismatch for test %d Stdin. Got %v want %v", i, val[j].Stdin, tc.expected[j].Stdin)
289 | 			}
290 | 			if val[j].Stdout != tc.expected[j].Stdout {
291 | 				t.Fatalf("Mismatch for test %d Stdout. Got %v want %v", i, val[j].Stdout, tc.expected[j].Stdout)
292 | 			}
293 | 			for k, _ := range val[j].Args {
294 | 			if val[j].Args[k] != tc.expected[j].Args[k] {
295 | 				t.Fatalf("Mismatch for test %d. Got %v want %v", i, val[j].Args[k], tc.expected[j].Args[k])
296 | 			}
297 | 			}
298 | 		}
299 | 	}
300 | }
301 | ```
302 | 
303 | Run our tests and..
304 | 
305 | ```sh
306 | --- FAIL: TestParseCommands (0.00s)
307 | 	tokenize_test.go:55: Unexpected number of ParsedCommands in test 1. Got [{[]  }] want [{[ls]  } {[cat]  }]
308 | FAIL
309 | FAIL	github.com/driusan/gosh	0.005s
310 | 
311 | ```
312 | 
313 | It's a good thing we wrote those tests, since it fails on the most basic
314 | case. It turns out we only set command.Args in the IsSpecial loop, which
315 | isn't getting triggered for the last iteration. Let's add an or to check if
316 | it's the last element, and do the same for the pipe statement while we're at it
317 | 
318 | We also forgot to reset currentCmd after a pipe, so let's take care of that
319 | 
320 | ### "ParseCommands Implementation"
321 | ```go
322 | // Keep track of the current command being built
323 | var currentCmd ParsedCommand
324 | // Keep array of all commands that have been built, so we can create the
325 | // pipeline
326 | var allCommands []ParsedCommand
327 | // Keep track of where this command started in parsed, so that we can build
328 | // currentCommand.Args when we find a special token.
329 | var lastCommandStart = 0
330 | // Keep track of if we've found a special token such as < or >, so that
331 | // we know if currentCmd.Args has already been populated.
332 | var foundSpecial bool
333 | for i, t := range tokens {
334 | 	if t.IsSpecial() || i == len(tokens)-1 {
335 | 		if foundSpecial == false {
336 | 			// Convert from Token to string. If it's a pipe, we want
337 | 			// to strip the '|' token, if it's the last token, we
338 | 			// don't want to strip anything.
339 | 			var slice []Token
340 | 			if i == len(tokens)-1 {
341 | 				slice = tokens[lastCommandStart:]
342 | 			} else {
343 | 				slice = tokens[lastCommandStart:i]
344 | 			}
345 | 			for _, t := range slice {
346 | 				currentCmd.Args = append(currentCmd.Args, string(t))
347 | 			}
348 | 		}
349 | 		foundSpecial = true
350 | 	}
351 | 	if t.IsPipe() || i == len(tokens)-1 {
352 | 		allCommands = append(allCommands, currentCmd)
353 | 		lastCommandStart = i+1
354 | 		foundSpecial = false
355 | 		currentCmd = ParsedCommand{}
356 | 	}
357 | }
358 | return allCommands
359 | ```
360 | 
361 | and now our tests pass. Let's take care of `<` and `>` while we're
362 | in this code. We'll just do it by adding some extra booleans to
363 | track if the next token is for redirecting Stdin or Stdout.
364 | 
365 | ### "ParseCommands Implementation"
366 | ```go
367 | // Keep track of the current command being built
368 | var currentCmd ParsedCommand
369 | // Keep array of all commands that have been built, so we can create the
370 | // pipeline
371 | var allCommands []ParsedCommand
372 | // Keep track of where this command started in parsed, so that we can build
373 | // currentCommand.Args when we find a special token.
374 | var lastCommandStart = 0
375 | // Keep track of if we've found a special token such as < or >, so that
376 | // we know if currentCmd.Args has already been populated.
377 | var foundSpecial bool
378 | var nextStdin, nextStdout bool
379 | for i, t := range tokens {
380 | 	if nextStdin {
381 | 		currentCmd.Stdin = string(t)
382 | 		nextStdin = false
383 | 	}
384 | 	if nextStdout {
385 | 		currentCmd.Stdout = string(t)
386 | 		nextStdout = false
387 | 	}
388 | 	if t.IsSpecial() || i == len(tokens)-1 {
389 | 		if foundSpecial == false {
390 | 			// Convert from Token to string
391 | 			var slice []Token
392 | 			if i == len(tokens)-1 {
393 | 				slice = tokens[lastCommandStart:]
394 | 			} else {
395 | 				slice = tokens[lastCommandStart:i]
396 | 			}
397 | 
398 | 			for _, t := range slice {
399 | 				currentCmd.Args = append(currentCmd.Args, string(t))
400 | 			}
401 | 		}
402 | 		foundSpecial = true
403 | 	}
404 | 	if t.IsStdinRedirect() {
405 | 		nextStdin = true
406 | 	}
407 | 	if t.IsStdoutRedirect() {
408 | 		nextStdout = true
409 | 	}
410 | 	if t.IsPipe() || i == len(tokens)-1 {
411 | 		allCommands = append(allCommands, currentCmd)
412 | 		lastCommandStart = i+1
413 | 		foundSpecial = false
414 | 		currentCmd = ParsedCommand{}
415 | 	}
416 | }
417 | return allCommands
418 | ```
419 | 
420 | We'll add some test cases to be sure:
421 | 
422 | ### "Other ParseCommands Test Cases"
423 | ```go
424 | {
425 | 	[]Token{"ls", ">", "cat"},
426 | 	[]ParsedCommand{
427 | 		ParsedCommand{[]string{"ls"}, "", "cat"},
428 | 	},
429 | },
430 | {
431 | 	[]Token{"ls", "<", "cat"},
432 | 	[]ParsedCommand{
433 | 		ParsedCommand{[]string{"ls"}, "cat", ""},
434 | 	},
435 | },
436 | {
437 | 	[]Token{"ls", ">", "foo", "<", "bar", "|", "cat", "hello", ">", "x", "|", "tee"},
438 | 	[]ParsedCommand{
439 | 		ParsedCommand{[]string{"ls"}, "bar", "foo"},
440 | 		ParsedCommand{[]string{"cat", "hello"}, "", "x"},
441 | 		ParsedCommand{[]string{"tee"}, "", ""},
442 | 	},
443 | },
444 | ```
445 | 
446 | And the tests still pass, so we seem to be alright.
447 | 
448 | That leaves us with building the commands, connecting their
449 | stdin and stdout pipes, and then running the whole thing.
450 | 
451 | ### Building the pipeline
452 | 
453 | We used to do this:
454 | 
455 | ```go
456 | cmd := exec.Command(parsed[0], args...)
457 | cmd.Stdin = os.Stdin
458 | cmd.Stdout = os.Stdout
459 | cmd.Stderr = os.Stderr
460 | 
461 | return cmd.Run()
462 | ```
463 | 
464 | which was nice and simple for a single command. Now we need to,
465 | at a minimum, range over all the commands and create different processes
466 | for each one, then set up their stdin and stdout pipes.
467 | 
468 | ### "Build pipeline and execute"
469 | ```go
470 | var cmds []*exec.Cmd
471 | for i, c := range commands {
472 | 	if len(c.Args) == 0 {
473 | 		// This should have never happened, there is
474 | 		// no command, but let's avoid panicing.
475 | 		continue
476 | 	}
477 | 	newCmd := exec.Command(c.Args[0], c.Args[1:]...)
478 | 	newCmd.Stderr = os.Stderr
479 | 	cmds = append(cmds, newCmd)
480 | 
481 | 	<<<Hookup stdin and stdout pipes>>>
482 | }
483 | 
484 | <<<Start Processes and Wait>>>
485 | ```
486 | 
487 | How do we hookup the pipes? If there was a `<` or `>` redirect, it's easy, we
488 | just open the file and replace the newCmd.Stdin/Stdout, overwriting the os.Stdin
489 | that we just set it to. Otherwise, we can use os variant we just set it up to.
490 | 
491 | ### "Hookup stdin and stdout pipes"
492 | ```go
493 | <<<Hookup STDIN>>>
494 | <<<Hookup STDOUT>>>
495 | ```
496 | 
497 | ### "Hookup STDIN"
498 | ```go
499 | // If there was an Stdin specified, use it.
500 | if c.Stdin != "" {
501 | 	// Open the file to convert it to an io.Reader
502 | 	if f, err := os.Open(c.Stdin); err == nil {
503 | 		newCmd.Stdin = f
504 | 		defer f.Close()
505 | 	}
506 | } else {
507 | 	// There was no Stdin specified, so 
508 | 	// connect it to the previous process in the
509 | 	// pipeline if there is one, the first process
510 | 	// still uses os.Stdin
511 | 	if i > 0 {
512 | 		pipe, err := cmds[i-1].StdoutPipe()
513 | 		if err != nil {
514 | 			continue
515 | 		}
516 | 		newCmd.Stdin = pipe
517 | 	} else {
518 | 		newCmd.Stdin = os.Stdin
519 | 	}
520 | }
521 | ```
522 | 
523 | ### "Hookup STDOUT"
524 | ```go
525 | // If there was a Stdout specified, use it.
526 | if c.Stdout != "" {
527 | 	// Create the file to convert it to an io.Reader
528 | 	if f, err := os.Create(c.Stdout); err == nil {
529 | 		newCmd.Stdout = f
530 | 		defer f.Close()
531 | 	}
532 | } else {
533 | 	// There was no Stdout specified, so 
534 | 	// connect it to the previous process in the
535 | 	// unless it's the last command in the pipeline,
536 | 	// which still uses os.Stdout
537 | 	if i == len(commands)-1  {
538 | 		newCmd.Stdout = os.Stdout
539 | 	}
540 | }
541 | ```
542 | 
543 | ### "Start Processes and Wait"
544 | ```go
545 | for _, c := range cmds {
546 | 	c.Start()
547 | }
548 | return cmds[len(cmds)-1].Wait()
549 | ```
550 | 


--------------------------------------------------------------------------------
/Prompts.md:
--------------------------------------------------------------------------------
  1 | # Prompts
  2 | 
  3 | We've went quite far with our trusty ">" prompt, but now we've done enough that
  4 | we might want to start using this as our login shell.
  5 | 
  6 | We'd like customizable prompts. We can start with just using $PROMPT as our
  7 | prompt if it exists, and falling back on our trusty old ">". While we're changing
  8 | things, we should probably print our prompt to `os.Stderr`, which is more
  9 | appropriate than Stdout for status-y type things like prompts or progress.
 10 | 
 11 | # "PrintPrompt Implementation"
 12 | ```go
 13 | if p := os.Getenv("PROMPT"); p != "" {
 14 | 	fmt.Fprintf(os.Stderr, "\n%s", p)
 15 | } else {
 16 | 	fmt.Fprintf(os.Stderr, "\n> ")
 17 | }
 18 | ```
 19 | 
 20 | And now if we wanted we could type `set PROMPT $` to make our shell look like
 21 | sh.. except we can't, because the `$` gets treated as an environment variable
 22 | by our parser before it gets to "set".
 23 | 
 24 | We can start by using the standard Go `os.ExpandEnv` function in our replacement
 25 | instead of our naive loop that just replaced any tokens that start with '$' with
 26 | the environment variable. This will also have the benefit of making our parser a
 27 | little more standard, and allowing us to use environment variables inside of
 28 | tokens, such as `$GOPATH/bin` too.
 29 | 
 30 | ### "Replace environment variables in command"
 31 | ```go
 32 | args := make([]string, 0, len(parsed))
 33 | for _, val := range parsed[1:] {
 34 | 	args = append(args, os.ExpandEnv(val))
 35 | }
 36 | ```
 37 | 
 38 | But that doesn't get us very far, either, because we still can't do anything
 39 | dynamic. We can try setting a `$` environment variable that evaluates to the
 40 | string "$" as a way to try escaping "$" in the shell and then something like
 41 | "set PROMPT $$PWD>" would theoretically set the prompt variable to the string
 42 | "$PWD>", but we'd have to hope that the ExpandEnv interprets $$PWD as ($$)PWD
 43 | and not $($PWD). Let's give it a shot anyways, but keeping in mind that if it
 44 | works we're depending on undocumented behaviour that may change in a future
 45 | version of Go.
 46 | 
 47 | ### "Initialize Terminal" +=
 48 | ```go
 49 | os.Setenv("$", "$")
 50 | ```
 51 | 
 52 | It *seems* to work, so let's improve our PrintPrint implementation to dynamically
 53 | expand variables that were set in the PROMPT variable too
 54 | 
 55 | # "PrintPrompt Implementation"
 56 | ```go
 57 | if p := os.Getenv("PROMPT"); p != "" {
 58 | 	fmt.Fprintf(os.Stderr, "\n%s", os.ExpandEnv(p))
 59 | } else {
 60 | 	fmt.Fprintf(os.Stderr, "\n> ")
 61 | }
 62 | ```
 63 | 
 64 | We'd now be able to do something like `set PROMPT '$$PWD>'` to get the current
 65 | working directory in our prompt, except that our cd implementation doesn't
 66 | set PWD. Let's fix it to set both 
 67 | 
 68 | ### "Handle cd command"
 69 | ```go
 70 | if len(args) == 0 {
 71 | 	return fmt.Errorf("Must provide an argument to cd")
 72 | }
 73 | old, _ := os.Getwd()
 74 | err := os.Chdir(args[0])
 75 | if err == nil {
 76 | 	new, _ := os.Getwd()
 77 | 	os.Setenv("PWD", new)
 78 | 	os.Setenv("OLDPWD", old)
 79 | }
 80 | return err
 81 | ```
 82 | 
 83 | There, now we can even do something like set PROMPT '$$PWD:$$?> ' to get the
 84 | last return code too.
 85 | 
 86 | What would be great though, was if we could use a '!' prefix to signify running
 87 | a command to run in order to get the prompt, similarly to how our tab completion
 88 | works.
 89 | 
 90 | It shouldn't be too hard to add a similar check to run the command with its
 91 | standard out being directed to stderr if our PROMPT variable starts with a '!'.
 92 | We could even expand the environment variables in the same way we're already
 93 | doing.
 94 | 
 95 | # "PrintPrompt Implementation"
 96 | ```go
 97 | if p := os.Getenv("PROMPT"); p != "" {
 98 | 	if len(p) > 1 && p[0] == '!' {
 99 | 		<<<Run command for prompt>>>
100 | 	} else {
101 | 		fmt.Fprintf(os.Stderr, "\n%s", os.ExpandEnv(p))
102 | 	}
103 | } else {
104 | 	fmt.Fprintf(os.Stderr, "\n> ")
105 | }
106 | ```
107 | 
108 | ### "Run command for prompt"
109 | ```go
110 | input := os.ExpandEnv(p[1:])
111 | split := strings.Fields(input)
112 | cmd := exec.Command(split[0], split[1:]...)
113 | cmd.Stdout = os.Stderr
114 | if err := cmd.Run(); err != nil {
115 | 	// Fall back on our standard prompt, with a warning.
116 | 	fmt.Fprintf(os.Stderr, "\nInvalid prompt command\n> ")
117 | }
118 | ```
119 | 
120 | But `Run` will return an err if the command exits with a non-zero error status,
121 | which may have nothing to do with our prompt. If it fails to run, the error is
122 | of type *ExitError according to the Run() documentation. So let's only print
123 | our warning if the error returned is of that type.
124 | 
125 | ### "Run command for prompt"
126 | ```go
127 | input := os.ExpandEnv(p[1:])
128 | split := strings.Fields(input)
129 | cmd := exec.Command(split[0], split[1:]...)
130 | cmd.Stdout = os.Stderr
131 | if err := cmd.Run(); err != nil {
132 | 	if _, ok := err.(*exec.ExitError); !ok {
133 | 		// Fall back on our standard prompt, with a warning.
134 | 		fmt.Fprintf(os.Stderr, "\nInvalid prompt command\n> ")
135 | 	}
136 | }
137 | ```
138 | 
139 | Now that we're customizing prompts, we might notice that if we set a prompt in
140 | our startup script, the first prompt gets printed before our `~/.goshrc` script
141 | is sourced. Let's add it to Initialize Shell
142 | 
143 | ### "Initialize Shell" += 
144 | ```go
145 | PrintPrompt()
146 | ```
147 | 
148 | and take it out of Initialize Terminal. (We'll have to do a little refactoring
149 | of our blocks that we probably should have done upfront.)
150 | 
151 | ### "Initialize Terminal"
152 | ```go
153 | // Initialize the terminal
154 | t, err := term.Open("/dev/tty")
155 | if err != nil {
156 | 	panic(err)
157 | }
158 | // Restore the previous terminal settings at the end of the program
159 | defer t.Restore()
160 | t.SetCbreak()
161 | terminal = t
162 | 
163 | <<<Create SIGCHLD chan>>>
164 | <<<Ignore certain signal types>>>
165 | os.Setenv("$", "$")
166 | ```
167 | 
168 | Now, we can create write prompts in any language of our choosing, as long as
169 | we can print to standard out in our language.
170 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # gosh Shell
  2 | 
  3 | This is an attempt to write a simple UNIX shell using literate programming. We'll
  4 | be using Go to write it, because I like Go.
  5 | 
  6 | (I intend to formalize the literate programming syntax I'm using with
  7 | markdown later, but it should be fairly straight forward. A header immediately
  8 | before a code block in quotation marks is a name for that code block. It can be
  9 | referenced in other code blocks as `<<<<name>>>>`. A `+=` at the end of the
 10 | header means append to the code block, don't replace it. A header without
 11 | quotation marks means the code block should be the contents of that filename.)
 12 | 
 13 | ## What does a simple shell do?
 14 | 
 15 | A simple shell needs to do a few things.
 16 | 
 17 | 1. Read user input.
 18 | 2. Interpret the user input.
 19 | 3. Execute the user's input.
 20 | 
 21 | And repeat, until the user inputs something like an `exit` command. A good
 22 | shell does much more (like tab completion, file globbing, etc) but for now
 23 | we'll stick to the absolute basics.
 24 | 
 25 | ## Main Body
 26 | 
 27 | We'll start with the main loop. Most Go programs have a file that looks
 28 | something like this:
 29 | 
 30 | ### main.go
 31 | ```go
 32 | package main
 33 | 
 34 | import (
 35 | 	<<<main.go imports>>>
 36 | )
 37 | 
 38 | <<<main.go globals>>>
 39 | 
 40 | <<<main.go funcs>>>
 41 | ```
 42 | 
 43 | ### "main.go funcs"
 44 | ```go
 45 | func main() {
 46 | 	<<<mainbody>>>
 47 | }
 48 | ```
 49 | Our main body is going to be a loop that repeatedly reads from `os.Stdin`.
 50 | This means that we should probably start by adding `os` to the import list.
 51 | 
 52 | ### "main.go imports"
 53 | ```go
 54 | "os"
 55 | ```
 56 | 
 57 | And we'll start with a loop that just repeatedly reads a rune from the user.
 58 | `*os.File` (`os.Stdin`'s type) doesn't support the ReadRune interface, but
 59 | fortunately the `bufio` package provides a wrapper which allows us to convert
 60 | any `io.Reader` into a RuneReader, so let's import that too.
 61 | 
 62 | ### "main.go imports" +=
 63 | ```go
 64 | "bufio"
 65 | ```
 66 | 
 67 | Then the basis of our main body becomes a loop that initializes and repeatedly
 68 | reads a rune from `os.Stdio`. For now, we'll just print it and see how it goes:
 69 | 
 70 | ### "mainbody"
 71 | ```go
 72 | r := bufio.NewReader(os.Stdin)
 73 | for {
 74 | 	c, _, err := r.ReadRune()
 75 | 	if err != nil {
 76 | 		panic(err)
 77 | 	}
 78 | 	print(c)
 79 | }
 80 | ```
 81 | 
 82 | The problem with this, is that `os.Stdin` sends things to the underlying `io.Reader`
 83 | one line at a time, and doesn't provide a way to change this. It turns out
 84 | that there's no way in the standard library to force it to send one character
 85 | at a time. Luckily for us, the third party package `github.com/pkg/term` *does*
 86 | provide a way to manipulate the settings of POSIX ttys, which is all we need
 87 | to support. So instead, let's import that so that we can convert the terminal
 88 | to raw mode. (We'll still use bufio for the simplicity of ReadRune.)
 89 | 
 90 | (We'll actually use CbreakMode(), which is like raw mode in that it'll send
 91 | us 1 key at a time, but unlike raw mode in that special command sequences
 92 | will still be interpreted.)
 93 | 
 94 | ### "main.go imports"
 95 | ```go
 96 | "github.com/pkg/term"
 97 | "bufio"
 98 | ```
 99 | 
100 | ### "mainbody"
101 | ```go
102 | // Initialize the terminal
103 | t, err := term.Open("/dev/tty")
104 | if err != nil {
105 | 	panic(err)
106 | }
107 | // Restore the previous terminal settings at the end of the program
108 | defer t.Restore()
109 | t.SetCbreak()
110 | r := bufio.NewReader(t)
111 | for {
112 | 	c, _, err := r.ReadRune()
113 | 	if err != nil {
114 | 		panic(err)
115 | 	}
116 | 	println(c)
117 | }
118 | ```
119 | 
120 | ## The Command Loop
121 | 
122 | That's a good start, but we probably want to handle the rune that's read in a
123 | way other than just printing it. Let's keep track of what the current command
124 | read is in a string, and when a newline is pressed, call a function to handle
125 | the current command and reset the string. We'll declare a type for commands,
126 | and do a little refactoring, just to be proactive.
127 | 
128 | ### "mainbody"
129 | ```go
130 | <<<Initialize Terminal>>>
131 | <<<Command Loop>>>
132 | ```
133 | 
134 | ### "main.go globals"
135 | ```go
136 | type Command string
137 | ```
138 | 
139 | ### "Initialize Terminal"
140 | ```go
141 | // Initialize the terminal
142 | t, err := term.Open("/dev/tty")
143 | if err != nil {
144 | 	panic(err)
145 | }
146 | // Restore the previous terminal settings at the end of the program
147 | defer t.Restore()
148 | t.SetCbreak()
149 | ```
150 | 
151 | ### "Command Loop"
152 | ```go
153 | r := bufio.NewReader(t)
154 | var cmd Command
155 | for {
156 | 	c, _, err := r.ReadRune()
157 | 	if err != nil {
158 | 		panic(err)
159 | 	}
160 | 	switch c {
161 | 		case '\n':
162 | 			// The terminal doesn't echo in raw mode,
163 | 			// so print the newline itself to the terminal.
164 | 			fmt.Printf("\n")
165 | 			<<<Handle Command>>>
166 | 			cmd = ""
167 | 		default:
168 | 			fmt.Printf("%c", c)
169 | 			cmd += Command(c)
170 | 	}
171 | }
172 | ```
173 | 
174 | Okay, but there's a problem. Since we're getting sent one character
175 | at a time, when we get the error `exec: "ls\u007f": executable file not found in $PATH`
176 | 
177 | (0x7F is the ASCII code for "DEL". 0x08 is the code for "Backspace".)
178 | 
179 | Let's make both of those work as backspace.
180 | 
181 | ### "Command Loop"
182 | ```go
183 | r := bufio.NewReader(t)
184 | var cmd Command
185 | for {
186 | 	c, _, err := r.ReadRune()
187 | 	if err != nil {
188 | 		panic(err)
189 | 	}
190 | 	switch c {
191 | 		case '\n':
192 | 			// The terminal doesn't echo in raw mode,
193 | 			// so print the newline itself to the terminal.
194 | 			fmt.Printf("\n")
195 | 			<<<Handle Command>>>
196 | 			cmd = ""
197 | 		case '\u007f', '\u0008':
198 | 			<<<Handle Backspace>>>
199 | 		default:
200 | 			fmt.Printf("%c", c)
201 | 			cmd += Command(c)
202 | 	}
203 | }
204 | ```
205 | 
206 | How do we handle the backspace key? We want to cut the last
207 | character off cmd, and erase it from the screen. Let's try 
208 | printing '\u0008' and see if that erases the last character
209 | in Cbreak mode:
210 | 
211 | ### "Handle Backspace"
212 | ```go
213 | if len(cmd) > 0 {
214 | 	cmd = cmd[:len(cmd)-1]
215 | 	fmt.Printf("\u0008")
216 | }
217 | ```
218 | 
219 | It moves the cursor, but doesn't actually delete the character. There might be
220 | a more appropriate character to print, but for now we'll just print backspace,
221 | space to overwrite the character, and then backspace again.
222 | 
223 | ### "Handle Backspace"
224 | ```go
225 | if len(cmd) > 0 {
226 | 	cmd = cmd[:len(cmd)-1]
227 | 	fmt.Printf("\u0008 \u0008")
228 | }
229 | ```
230 | 
231 | ## Handling the Command
232 | Okay, so how do we handle the command? If it's the string "exit"
233 | we probably should exit. Otherwise, we'll want to execute it using
234 | the [`os.exec`](https://golang.org/pkg/os/exec/) package. We were
235 | proactive about declaring cmd as a type instead of a string, so
236 | we can just define some kind of HandleCmd() method on the type and
237 | call that.
238 | 
239 | ### "Handle Command"
240 | ```go
241 | if cmd == "exit" || cmd == "quit" {
242 | 	os.Exit(0)
243 | } else {
244 | 	cmd.HandleCmd();
245 | }
246 | ```
247 | 
248 | ### "main.go funcs" +=
249 | ```go
250 | <<<HandleCmd Implementation>>>
251 | ```
252 | 
253 | ### "HandleCmd Implementation"
254 | ```go
255 | func (c Command) HandleCmd() error {
256 | 	cmd := exec.Command(string(c))
257 | 	return cmd.Run()
258 | }
259 | ```
260 | 
261 | We'll need to add `os` and `os/exec` to our imports, while we're
262 | at it.
263 | 
264 | ### "main.go imports" +=
265 | ```go
266 | "os"
267 | "os/exec"
268 | ```
269 | 
270 | If we run it and try executing something, it doesn't seem to be working.
271 | What's going on? Let's print the error if it happens to find out.
272 | 
273 | ### "Handle Command"
274 | ```go
275 | if cmd == "exit" || cmd == "quit" {
276 | 	os.Exit(0)
277 | } else {
278 | 	err := cmd.HandleCmd()
279 | 	if err != nil {
280 | 		fmt.Fprintf(os.Stderr, "%v\n", err)
281 | 	}
282 | }
283 | ```
284 | 
285 | ### "main.go imports" +=
286 | ```go
287 | "fmt"
288 | ```
289 | 
290 | There's still no error (unless we just hit enter without entering anything,
291 | in which case it complains about not being able to execute "". (We should
292 | probably handle that as a special case, too.)) 
293 | 
294 | So what's going on with the lack of any output or error? It turns out if we look at the `os/exec.Command`
295 | documentation we'll see:
296 | 
297 | ```go
298 | // Stdout and Stderr specify the process's standard output and error.
299 | //
300 | // If either is nil, Run connects the corresponding file descriptor
301 | // to the null device (os.DevNull).
302 | //
303 | // If Stdout and Stderr are the same writer, at most one
304 | // goroutine at a time will call Write.
305 | Stdout io.Writer
306 | Stderr io.Writer
307 | ```
308 | 
309 | `Stdin` makes a similar claim. So let's hook those up to os.Stdin, os.Stdout
310 | and os.Stderr.
311 | 
312 | While we're at it, let's print a simple prompt.
313 | 
314 | So our new command handling code is:
315 | 
316 | ### "Handle Command"
317 | ```go
318 | if cmd == "exit" || cmd == "quit" {
319 | 	os.Exit(0)
320 | } else if cmd == "" {
321 | 	PrintPrompt()
322 | } else {
323 | 	err := cmd.HandleCmd()
324 | 	if err != nil {
325 | 		fmt.Fprintf(os.Stderr, "%v\n", err)
326 | 	}
327 | 	PrintPrompt()
328 | }
329 | ```
330 | 
331 | We need to define PrintPrompt() that we just used.
332 | 
333 | ### "main.go funcs" +=
334 | ```go
335 | func PrintPrompt() {
336 | 	<<<PrintPrompt Implementation>>>
337 | }
338 | ```
339 | 
340 | We'll start with a simple implementation that just prints a ">" so that we don't 
341 | get confused and think we're in a POSIX compliant sh prompt, then work on adding
342 | a better prompt later.
343 | 
344 | # "PrintPrompt Implementation"
345 | ```go
346 | fmt.Printf("\n> ")
347 | ```
348 | 
349 | 
350 | And we'll want to print it on startup too:
351 | 
352 | ### "Initialize Terminal" +=
353 | ```go
354 | PrintPrompt()
355 | ```
356 | 
357 | And our new HandleCmd() implementation.
358 | 
359 | ### "HandleCmd Implementation"
360 | ```go
361 | func (c Command) HandleCmd() error {
362 | 	cmd := exec.Command(string(c))
363 | 	cmd.Stdin = os.Stdin
364 | 	cmd.Stdout = os.Stdout
365 | 	cmd.Stderr = os.Stderr
366 | 
367 | 	return cmd.Run()
368 | }
369 | ```
370 | 
371 | Now we can finally run some commands from our shell!
372 | 
373 | But wait, when we run anything with arguments, we get an
374 | error: `exec: "ls -l": executable file not found in $PATH`.
375 |  exec is trying to run the command named "ls -l", not the
376 | command "ls" with the parameter "-l". 
377 | 
378 | 
379 | This is the signature from the GoDoc:
380 | 
381 | `func Command(name string, arg ...string) *Cmd`
382 | 
383 | *Not*
384 | 
385 | `func Command(command string) *Cmd`
386 | 
387 | We can use `Fields` function in the standard Go [`strings`](https://golang.org/pkg/strings)
388 | package to split a string on any whitespace, which is basically what we want
389 | right now. There will be some problems with that (notably we won't be able 
390 | to enclose arguments in quotation marks if they contain whitespace), but at 
391 | least we won't have to write our own tokenizer.
392 | 
393 | While we're at it, the "$PATH" in the error message reminds me. If there *are*
394 | any arguments that start with a "$", we should probably expand that to the OS
395 | environment variable.
396 | 
397 | ### "HandleCmd Implementation"
398 | ```go
399 | func (c Command) HandleCmd() error {
400 | 	parsed := strings.Fields(string(c))
401 | 	if len(parsed) == 0 {
402 | 		// There was no command, it's not an error, the user just hit
403 | 		// enter.
404 | 		PrintPrompt()
405 | 		return nil
406 | 	}
407 | 
408 | 	var args []string
409 | 	for _, val := range parsed[1:] {
410 | 		if val[0] == '$' {
411 | 			args = append(args, os.Getenv(val[1:])
412 | 		} else {
413 | 			args = append(args, val)
414 | 		}
415 | 	}
416 | 	cmd := exec.Command(parsed[0], args...)
417 | 	cmd.Stdin = os.Stdin
418 | 	cmd.Stdout = os.Stdout
419 | 	cmd.Stderr = os.Stderr
420 | 
421 | 	return cmd.Run()
422 | }
423 | ```
424 | 
425 | ### "main.go imports" +=
426 | ```go
427 | "strings"
428 | ```
429 | 
430 | There's one other command that needs to be implemented internally: `cd`. Each
431 | program in unix contains it's own working directory. When it's spawned, it
432 | inherits its parent's working directory. If `cd` were implemented as an
433 | external program, the new directory would never make it back to the
434 | parent (our shell.) It's fairly easy to change the working directory,
435 | we just add a check after we've parsed the args if the command is "cd", and
436 | call `os.Chdir` instead of `exec.Command` as appropriate.
437 | 
438 | ### "HandleCmd Implementation"
439 | ```go
440 | func (c Command) HandleCmd() error {
441 | 	parsed := strings.Fields(string(c))
442 | 	if len(parsed) == 0 {
443 | 		// There was no command, it's not an error, the user just hit
444 | 		// enter.
445 | 		PrintPrompt()
446 | 		return nil
447 | 	}
448 | 
449 | 	var args []string
450 | 	for _, val := range parsed[1:] {
451 | 		if val[0] == '$' {
452 | 			args = append(args, os.Getenv(val[1:]))
453 | 		} else {
454 | 			args = append(args, val)
455 | 		}
456 | 	}
457 | 
458 | 	if parsed[0] == "cd" {
459 | 		if len(args) == 0 {
460 | 			return fmt.Errorf("Must provide an argument to cd")
461 | 		}
462 | 		return os.Chdir(args[0])
463 | 
464 | 	}
465 | 
466 | 	cmd := exec.Command(parsed[0], args...)
467 | 	cmd.Stdin = os.Stdin
468 | 	cmd.Stdout = os.Stdout
469 | 	cmd.Stderr = os.Stderr
470 | 
471 | 	return cmd.Run()
472 | }
473 | ```
474 | 
475 | ### Handling EOFs
476 | 
477 | There's still one minor noticable bug. If we hit `^D` on an empty line, it should
478 | be treated as an EOF instead of adding the character `0x04`. (And
479 | if it's not an empty line, we probably still shouldn't add it to
480 | the command.)
481 | 
482 | ### "Command Loop"
483 | ```go
484 | r := bufio.NewReader(t)
485 | var cmd Command
486 | for {
487 | 	c, _, err := r.ReadRune()
488 | 	if err != nil {
489 | 		panic(err)
490 | 	}
491 | 	switch c {
492 | 		case '\n':
493 | 			// The terminal doesn't echo in raw mode,
494 | 			// so print the newline itself to the terminal.
495 | 			fmt.Printf("\n")
496 | 			<<<Handle Command>>>
497 | 			cmd = ""
498 | 		case '\u0004':
499 | 			if len(cmd) == 0 {
500 | 				os.Exit(0)
501 | 			}
502 | 		case '\u007f', '\u0008':
503 | 			<<<Handle Backspace>>>
504 | 		default:
505 | 			fmt.Printf("%c", c)
506 | 			cmd += Command(c)
507 | 	}
508 | }
509 | ```
510 | 
511 | And now.. hooray! We have a simple shell that works! We should add tab completion,
512 | a smarter tokenizer, and a lot of other features if we were going to use this
513 | every day, but at least we have a proof-of-concept, and maybe you learned something
514 | by doing it.
515 | 
516 | If there's any other features you'd like to see added, feel free to either
517 | create a pull request telling the story of how to you'd do it, or just create
518 | an issue on GitHub and see if someone else does. Feel free to also file bug
519 | reports in either the code or prose.
520 | 
521 | The Tokenization.md file builds on this and improves on the tokenization by
522 | adding support for string literals (with spaces) in arguments.
523 | 
524 | TabCompletion.md builds on Tokenization.md to add rudimentary command and file
525 | tab completion to the shell.
526 | 
527 | Piping.md adds support for stdin/stdout redirection and piping processes
528 | together with `|`.
529 | 
530 | The final result of putting this all together after running `go fmt` is in the
531 | accompanying `*.go` files in this repo, so it should be go gettable.
532 | 


--------------------------------------------------------------------------------
/TabCompletion.md:
--------------------------------------------------------------------------------
  1 | # Tab Completion
  2 | 
  3 | We can't have a modern shell without tab completion. Tab completion
  4 | is what separates us from the dumb terminals. A shell with good
  5 | tab completion adds some form of context-sensitive, easy, and 
  6 | customizable tab completion.
  7 | 
  8 | We'll want, at a minimum, to support autocompletion of commands
  9 | (for the first command) and filenames (for any other command.) Maybe
 10 | in addition to that, we can add support for regular expressions
 11 | that allow the user to add to the suggestions.
 12 | 
 13 | Let's start with the first two.
 14 | 
 15 | # Command Completion
 16 | 
 17 | We need to start actually adding support for the '\t' character in
 18 | our command loop. Recall that by the end of our basic implementation,
 19 | it looked like this: 
 20 | 
 21 | ### "Command Loop"
 22 | r := bufio.NewReader(t)
 23 | var cmd Command
 24 | for {
 25 | 	c, _, err := r.ReadRune()
 26 | 	if err != nil {
 27 | 		panic(err)
 28 | 	}
 29 | 	switch c {
 30 | 		case '\n':
 31 | 			// The terminal doesn't echo in raw mode,
 32 | 			// so print the newline itself to the terminal.
 33 | 			fmt.Printf("\n")
 34 | 			<<<Handle Command>>>
 35 | 			cmd = ""
 36 | 		case '\u0004':
 37 | 			if len(cmd) == 0 {
 38 | 				os.Exit(0)
 39 | 			}
 40 | 		case '\u007f', '\u0008':
 41 | 			<<<Handle Backspace>>>
 42 | 		default:
 43 | 			fmt.Printf("%c", c)
 44 | 			cmd += Command(c)
 45 | 	}
 46 | }
 47 | 
 48 | We'll start by adding a '\t' case, and for now just assume that
 49 | we'll implement a method called "Complete()" on the `Command` class.
 50 | If it returns an error, we should print it. (In fact, let's do it
 51 | in the non-zero case of '^D' too.
 52 | 
 53 | ### "Command Loop"
 54 | ```go
 55 | r := bufio.NewReader(t)
 56 | var cmd Command
 57 | for {
 58 | 	c, _, err := r.ReadRune()
 59 | 	if err != nil {
 60 | 		fmt.Fprintf(os.Stderr, "%v\n", err)
 61 | 		continue
 62 | 	}
 63 | 	switch c {
 64 | 		case '\n':
 65 | 			// The terminal doesn't echo in raw mode,
 66 | 			// so print the newline itself to the terminal.
 67 | 			fmt.Printf("\n")
 68 | 
 69 | 			<<<Handle Command>>>
 70 | 			cmd = ""
 71 | 		case '\u0004':
 72 | 			if len(cmd) == 0 {
 73 | 				os.Exit(0)
 74 | 			}
 75 | 			err := cmd.Complete()
 76 | 			if err != nil {
 77 | 				fmt.Fprintf(os.Stderr, "%v\n", err)
 78 | 			}
 79 | 
 80 | 		case '\u007f', '\u0008':
 81 | 			<<<Handle Backspace>>>
 82 | 		case '\t':
 83 | 			err := cmd.Complete()
 84 | 			if err != nil {
 85 | 				fmt.Fprintf(os.Stderr, "%v\n", err)
 86 | 			}
 87 | 		default:
 88 | 			fmt.Printf("%c", c)
 89 | 			cmd += Command(c)
 90 | 	}
 91 | }
 92 | ```
 93 | 
 94 | We'll need to define Command.Complete() to make sure we don't have a compile
 95 | error, too. We'll put it in a new file.
 96 | 
 97 | ### completion.go
 98 | ```go
 99 | package main
100 | 
101 | import (
102 | 	<<<completion.go imports>>>
103 | )
104 | 
105 | <<<completion.go globals>>>
106 | 
107 | func (c *Command) Complete() error {
108 | 	<<<AutoCompletion Implementation>>>
109 | }
110 | 
111 | <<<other completion.go functions>>>
112 | ```
113 | 
114 | ### "completion.go globals"
115 | ```go
116 | ```
117 | 
118 | For now, we'll start with an implementation that just does nothing but return
119 | no error.
120 | 
121 | ### "AutoCompletion Implementation"
122 | ```go
123 | return nil
124 | ```
125 | 
126 | Okay, now that it compiles, what do we *actually* need to do for tab
127 | completion?
128 | 
129 | We should probably start by tokenizing the command, so we know if we're
130 | completing a command or a file.
131 | 
132 | ### "AutoCompletion Implementation"
133 | ```go
134 | tokens := c.Tokenize()
135 | var suggestions []string
136 | var base string
137 | switch len(tokens) {
138 | case 0:
139 | 	base = ""
140 | 	suggestions = CommandSuggestions(base)
141 | case 1:
142 | 	base = tokens[0]
143 | 	suggestions = CommandSuggestions(base)
144 | default:
145 | 	base = tokens[len(tokens)-1]
146 | 	suggestions = FileSuggestions(base)
147 | }
148 | ```
149 | 
150 | What we do with the suggestions that we get back (don't worry,
151 | we'll define the functions we just called in a bit) is going
152 | to depend on the number of suggestions. If there's none, we do
153 | nothing (except maybe print a `BEL` character as a warning).
154 | If there's one, we'll use it, and if there's more than one, we'll
155 | print them.
156 | 
157 | (We also still need to return something.)
158 | 
159 | ### "AutoCompletion Implementation" +=
160 | ```go
161 | switch len(suggestions) {
162 | case 0:
163 | 	fmt.Printf("\u0007")
164 | case 1:
165 | 	<<<Complete Suggestion>>>
166 | default:
167 | 	<<<Display Suggestions>>>
168 | }
169 | return nil
170 | ```
171 | 
172 | 
173 | So, we said we'd define the functions we just used:
174 | 
175 | ### "other completion.go functions"
176 | ```go
177 | func CommandSuggestions(base string) []string {
178 | 	<<<Command Suggestions Implementation>>>
179 | }
180 | 
181 | func FileSuggestions(base string) []string {
182 | 	<<<File Suggestions Implementation>>>
183 | }
184 | ```
185 | 
186 | We should also import the `fmt` package that we just used:
187 | 
188 | ### "completion.go imports"
189 | ```go
190 | "fmt"
191 | ```
192 | 
193 | ## Command Suggestions
194 | 
195 | Let's start with the command suggestions. We'll need to get the $PATH
196 | variable, parse it, get a list of the contents of each directory, and
197 | then go through them and see if any have "base" as a prefix. Ideally
198 | we would cache the results, but for now we'll just redo it every time
199 | and see if the performance is unusable.
200 | 
201 | ### "Command Suggestions Implementation"
202 | ```go
203 | paths := strings.Split(os.Getenv("PATH"), ":")
204 | var matches []string
205 | for _, path := range paths {
206 | 	<<<Check For Command Completion in path>>>
207 | }
208 | return matches
209 | ```
210 | 
211 | How are we going to look into the paths? We can use `io/ioutil.ReadDir` to get
212 | a list of files, so let's add that (and the things we just used) to the import
213 | list.
214 | 
215 | ### "completion.go imports" +=
216 | ```go
217 | "os"
218 | "strings"
219 | "io/ioutil"
220 | ```
221 | 
222 | ### "Check For Command Completion in path"
223 | ```go
224 | // We don't care if there's an invalid path in $PATH, so ignore
225 | // the error.
226 | files, _ := ioutil.ReadDir(path)
227 | for _, file := range files {
228 | 	if name := file.Name(); strings.HasPrefix(name, base) {
229 | 		matches = append(matches, name)
230 | 	}
231 | }
232 | ```
233 | 
234 | For now, we'll just leave a stub for filename suggestions, so that things
235 | compile.
236 | ### "File Suggestions Implementation"
237 | ```go
238 | return nil
239 | ```
240 | 
241 | But before we test it, we'll at least want to declare Display Suggestions
242 | and Complete Suggestions
243 | 
244 | Display is easy.
245 | 
246 | ### "Display Suggestions"
247 | ```go
248 | fmt.Printf("\n%v\n", suggestions)
249 | ```
250 | 
251 | For completion, we need to delete the last token, and then add the new
252 | suggestion. Since the Command is a type of string, not a type of []string,
253 | we need to mimick the tokenization. We can strip the trailing whitespace,
254 | then strip the last token as a suffix, then add the new autocompleted command.
255 | 
256 | ### "Complete Suggestion"
257 | ```go
258 | suggest := suggestions[0]
259 | *c = Command(strings.TrimSpace(string(*c)))
260 | *c = Command(strings.TrimSuffix(string(*c), base))
261 | *c += Command(suggest)
262 | ```
263 | 
264 | We should also print the remaining part of the token that we just
265 | completed. Since we don't know if the autocomplete suggestions screwed up
266 | the cursor, we'll just print a new prompt with the completed command for now.
267 | We should make this smarter later.
268 | 
269 | ### "Complete Suggestion" +=
270 | ```go
271 | PrintPrompt()
272 | fmt.Printf("%s", *c)
273 | ```
274 | 
275 | Finally, we should implement the file name completion we said we were going
276 | to do for other parameters.
277 | 
278 | To do that, we'll use the standard `path/filepath` package.
279 | 
280 | ### "completion.go imports" +=
281 | ```go
282 | "path/filepath"
283 | ```
284 | 
285 | We'll try to just call filepath.Dir() on the token, and then use the same
286 | ReadDir() method we used above to try and find any files that match
287 | filepath.Base()
288 | 
289 | ### "File Suggestions Implementation"
290 | ```go
291 | filedir := filepath.Dir(base)
292 | fileprefix := filepath.Base(base)
293 | files, err := ioutil.ReadDir(filedir)
294 | if err != nil {
295 | 	return nil
296 | }
297 | var matches []string
298 | for _, file := range files {
299 | 	if name := file.Name(); strings.HasPrefix(name, fileprefix) {
300 | 		matches = append(matches, filedir + "/" + name)
301 | 	}
302 | }
303 | return matches
304 | ```
305 | 
306 | There's two problems with this implementation: if filedir is `/`, the
307 | slash gets doubled up when completed, and if base itself is a directory, it
308 | doesn't look into it. We'll want to share the files ranging logic when checking
309 | the base directory, so let's break that out into a new section.
310 | 
311 | ### "Check files for matches and return"
312 | ```go
313 | var matches []string
314 | for _, file := range files {
315 | 	if name := file.Name(); strings.HasPrefix(name, fileprefix) {
316 | 		<<<Append file match>>>
317 | 	}
318 | }
319 | return matches
320 | ```
321 | 
322 | The fix for `<<<Append file match>>>` is fairly straightforward.
323 | 
324 | ### "Append file match"
325 | ```go
326 | if filedir != "/" {
327 | 	matches = append(matches, filedir + "/" + name)
328 | } else {
329 | 	matches = append(matches, filedir + name)
330 | }
331 | ```
332 | 
333 | Then the suggestions implementation becomes:
334 | 
335 | ### "File Suggestions Implementation"
336 | ```go
337 | <<<Check base dir>>>
338 | 
339 | filedir := filepath.Dir(base)
340 | fileprefix := filepath.Base(base)
341 | files, err := ioutil.ReadDir(filedir)
342 | if err != nil {
343 | 	return nil
344 | }
345 | 
346 | <<<Check files for matches and return>>>
347 | ```
348 | 
349 | 
350 | 
351 | To check the base dir, we just need to open it as a directory before doing
352 | any other processing. If it opens successfully as a directory, we add
353 | matches and return before going into parsing base into diretory/file.
354 | 
355 | ### "Check base dir"
356 | ```go
357 | if files, err := ioutil.ReadDir(base); err == nil {
358 | 	// This was a directory, so use the empty string as a prefix.
359 | 	fileprefix := ""
360 | 	filedir := base
361 | 	<<<Check files for matches and return>>>
362 | }
363 | ```
364 | 
365 | Now, we should have a workable tab completion implementation.
366 | 


--------------------------------------------------------------------------------
/TabCompletionRevisited.md:
--------------------------------------------------------------------------------
   1 | # Improving Tab Completion
   2 | 
   3 | We already have a basic tab completion for filenames, but to be a really useful
   4 | shell it would be nice if we could customize the behaviour of it and add the
   5 | ability for users to have context-sensitive customizable tab completion. For
   6 | instance, if the command is `git`, autocompleting to filenames doesn't make much
   7 | sense. Better suggestions would be "add" or "commit" or "checkout".
   8 | 
   9 | Let's add a builtin command with a name like "autocomplete" to customize the
  10 | behaviour. (We also didn't have the infrastructure in our codes to easily add
  11 | builtins when we first did tab completion.)
  12 | 
  13 | How should our "autocomplete" builtin work? It needs a way to match the current
  14 | string (minus the last token, which is being completed..) against some pattern,
  15 | and then evaluate if it matched. Regexes seem like an obvious solution. Let's
  16 | tokenize the command, remove the last token, and then compare it against a regex.
  17 | If any custom autocompleter matched, we'll use those suggestions, otherwise we'll
  18 | fall back on the old behaviour. To start, we'll make the list of suggestions
  19 | parameters passed to "autocomplete" after the regex. 
  20 | 
  21 | So defining a suggestion might be something like
  22 | 
  23 | ```sh
  24 | > autocompete /^git/ add checkout commit
  25 | ```
  26 | 
  27 | In fact, we don't really need the normal regex slash delimitors since we're taking
  28 | the first parameter, and maybe we should just make the '^' implicit, because we'll
  29 | pretty much always want our regexes to start at the start of the command. That would give us
  30 | 
  31 | ```sh
  32 | > autocompete git add checkout commit
  33 | ```
  34 | 
  35 | which is pretty nice, but then again, maybe there is a use case for completion
  36 | suggetions that aren't anchored at the start of the command, so for now we'll
  37 | keep the `^` and ditch the `/`.
  38 | 
  39 | ## Implementing Regex Completion
  40 | 
  41 | Recall our previous auto completion implementation was:
  42 | 
  43 | ### "AutoCompletion Implementation"
  44 | ```go
  45 | tokens := c.Tokenize()
  46 | var suggestions []string
  47 | var base string
  48 | switch len(tokens) {
  49 | case 0:
  50 | 	base = ""
  51 | 	suggestions = CommandSuggestions(base)
  52 | case 1:
  53 | 	base = tokens[0]
  54 | 	suggestions = CommandSuggestions(base)
  55 | default:
  56 | 	base = tokens[len(tokens)-1]
  57 | 	suggestions = FileSuggestions(base)
  58 | }
  59 | 
  60 | switch len(suggestions) {
  61 | case 0:
  62 | 	// Print BEL to warn that there were no suggestions.
  63 | 	fmt.Printf("\u0007")
  64 | case 1:
  65 | 	<<<Complete Suggestion>>>
  66 | default:
  67 | 	<<<Display Suggestions>>>
  68 | }
  69 | return nil
  70 | ```
  71 | 
  72 | It's really only the default "FileSuggestions" in the first switch statement
  73 | that we're going to want to change right now. Instead, we'll do something like
  74 | 
  75 | ### "AutoCompletion Implementation"
  76 | ```go
  77 | tokens := c.Tokenize()
  78 | var suggestions []string
  79 | var base string
  80 | switch len(tokens) {
  81 | case 0:
  82 | 	base = ""
  83 | 	suggestions = CommandSuggestions(base)
  84 | case 1:
  85 | 	base = tokens[0]
  86 | 	suggestions = CommandSuggestions(base)
  87 | default:
  88 | 	<<<Check regex suggestions and break if found>>>
  89 | 	<<<Check file suggestions>>>
  90 | }
  91 | 
  92 | switch len(suggestions) {
  93 | case 0:
  94 | 	// Print BEL to warn that there were no suggestions.
  95 | 	fmt.Printf("\u0007")
  96 | case 1:
  97 | 	<<<Complete Suggestion>>>
  98 | default:
  99 | 	<<<Display Suggestions>>>
 100 | }
 101 | return nil
 102 | ```
 103 | 
 104 | ### "Check file suggestions"
 105 | ```go
 106 | base = tokens[len(tokens)-1]
 107 | suggestions = FileSuggestions(base)
 108 | ```
 109 | 
 110 | So how do we "Check regex suggestions and break if found"? We'll want to start
 111 | by importing the regex package that we know we're going to use.
 112 | 
 113 | ### "completion.go imports" +=
 114 | ```go
 115 | "regexp"
 116 | ```
 117 | 
 118 | And we know we're going to need a list of Regex->Suggestion mappings, so let's
 119 | define that.
 120 | 
 121 | ### "completion.go globals" +=
 122 | ```go
 123 | <<<Autocompletion Map>>>
 124 | ```
 125 | 
 126 | ### "Autocompletion Map"
 127 | ```go
 128 | var autocompletions map[regexp.Regexp][]Token
 129 | ```
 130 | 
 131 | Now, our check is pretty straight forward. We just range over the map, and
 132 | any regex that matches the current command gets added to suggestions (if the
 133 | last token matches the suggestion.) Then we'll break if we found any.
 134 | 
 135 | ### "Check regex suggestions and break if found"
 136 | ```go
 137 | firstpart := strings.Join(tokens[:len(tokens)-1], " ")
 138 | lasttoken := tokens[len(tokens)-1]
 139 | for re, resuggestions := range autocompletions {
 140 | 	if re.MatchString(firstpart) {
 141 | 		for _, val := range resuggestions {
 142 | 			if strings.HasPrefix(string(val), lasttoken) {
 143 | 				suggestions = append(suggestions, string(val))
 144 | 			}
 145 | 		}
 146 | 	}
 147 | }
 148 | 
 149 | if len(suggestions) > 0 {
 150 | 	break
 151 | }
 152 | ```
 153 | 
 154 | (Note that we know the length is at least 2, because of the position in the
 155 | switch statement.)
 156 | 
 157 | ### "completion.go imports" +=
 158 | ```go
 159 | "strings"
 160 | ```
 161 | 
 162 | Now we're getting a compile error: "invalid map key type regexp.Regexp". If
 163 | we look into it, we find that regexp.Regexp has a slice under the hood, which
 164 | can't be used as a map key in Go because their size isn't fixed. To get around
 165 | this, we'll just make our map a map of pointers to regexp.Regexpes.
 166 | 
 167 | ### "Autocompletion Map"
 168 | ```go
 169 | var autocompletions map[*regexp.Regexp][]Token
 170 | ```
 171 | 
 172 | That was suspiciously easy, but we can't use it yet without defining a way
 173 | to set them. Let's define our completion builtin. All we need to do is check
 174 | the arguments, create the map if it hasn't been created yet, and populate
 175 | the tokens.
 176 | 
 177 | ### "Builtin Commands" +=
 178 | ```go
 179 | case "autocomplete":
 180 | <<<AutoComplete Builtin Command>>>
 181 | ```
 182 | 
 183 | ### "AutoComplete Builtin Command"
 184 | ```go
 185 | <<<Check autocomplete usage>>>
 186 | <<<Create autocomplete map if nil>>>
 187 | <<<Add suggestions to map>>>
 188 | 
 189 | return nil
 190 | ```
 191 | 
 192 | ### "Check autocomplete usage"
 193 | ```go
 194 | if len(args) < 2 {
 195 | 	return fmt.Errorf("Usage: autocomplete regex value [more values...]")
 196 | }
 197 | ```
 198 | 
 199 | ### "Create autocomplete map if nil"
 200 | ```go
 201 | if autocompletions == nil {
 202 | 	autocompletions = make(map[*regexp.Regexp][]Token)
 203 | }
 204 | ```
 205 | 
 206 | ### "Add suggestions to map"
 207 | ```go
 208 | re, err := regexp.Compile(args[0])
 209 | if err != nil {
 210 | 	return err
 211 | }
 212 | 
 213 | for _, t := range args[1:] {
 214 | 	autocompletions[re] = append(autocompletions[re], Token(t))
 215 | }
 216 | ```
 217 | 
 218 | The builtin handler is in main.go, so we'll need to import regexp there too.
 219 | 
 220 | ### "main.go imports" +=
 221 | ```go
 222 | "regexp"
 223 | ```
 224 | 
 225 | Now, there's a problem where the completion isn't suggestion isn't removing the
 226 | part that was already typed, so 'git ad<tab>' is completing to 'git adadd'. In
 227 | our suggestion code from TabCompletion.md, it trims the variable "base" to
 228 | avoid that for files, so we should probably set the "base" variable to lasttoken
 229 | when we find a match too (or better yet, just re-use the `base` variable instead
 230 | of making a new "lasttoken" variable for the same purpose.)
 231 | 
 232 | ### "Check regex suggestions and break if found"
 233 | ```go
 234 | firstpart := strings.Join(tokens[:len(tokens)-1], " ")
 235 | base = tokens[len(tokens)-1]
 236 | for re, resuggestions := range autocompletions {
 237 | 	if re.MatchString(firstpart) {
 238 | 		for _, val := range resuggestions {
 239 | 			if strings.HasPrefix(string(val), base) {
 240 | 				suggestions = append(suggestions, string(val))
 241 | 			}
 242 | 		}
 243 | 	}
 244 | }
 245 | 
 246 | if len(suggestions) > 0 {
 247 | 	break
 248 | }
 249 | ```
 250 | 
 251 | In fact, there's a slight problem with our check. The way we've implemented it,
 252 | we need to type at least one character. If we just type "git" we won't get our
 253 | suggestions until we type at least one more letter.
 254 | 
 255 | To do fix that, we'll just checks. The first one will check the entire
 256 | command and use a base of "", and the second one will do what we've just done.
 257 | Since it's the same regex, we can do it in one loop, but we can only have either
 258 | whole command *or* subtoken matches, because of the "base" variable, so once
 259 | we find one we'll break out of our autocompletions loop to prevent the risk
 260 | of conflicts between an empty and non-empty base.
 261 | 
 262 | ### "Check regex suggestions and break if found"
 263 | ```go
 264 | firstpart := strings.Join(tokens[:len(tokens)-1], " ")
 265 | wholecmd := strings.Join(tokens, " ")
 266 | base = tokens[len(tokens)-1]
 267 | for re, resuggestions := range autocompletions {
 268 | 	if re.MatchString(wholecmd) {
 269 | 		for _, val := range resuggestions {
 270 | 			// There was no last token, to take the prefix of, so
 271 | 			// just suggest the whole val.
 272 | 			suggestions = append(suggestions, string(val))
 273 | 		}
 274 | 		base = " "
 275 | 		break
 276 | 	} else if re.MatchString(firstpart) {
 277 | 		for _, val := range resuggestions {
 278 | 			if strings.HasPrefix(string(val), base) {
 279 | 				suggestions = append(suggestions, string(val))
 280 | 			}
 281 | 		}
 282 | 	}
 283 | }
 284 | 
 285 | if len(suggestions) > 0 {
 286 | 	break
 287 | }
 288 | ```
 289 | 
 290 | In fact, we're going to want this behaviour for command suggestions too, so let's
 291 | add it to our switch.
 292 | 
 293 | ### "AutoCompletion Implementation"
 294 | ```go
 295 | tokens := c.Tokenize()
 296 | var suggestions []string
 297 | var base string
 298 | switch len(tokens) {
 299 | case 0:
 300 | 	<<<Check regex suggestions and break if found>>>
 301 | 	base = ""
 302 | 	suggestions = CommandSuggestions(base)
 303 | case 1:
 304 | 	<<<Check regex suggestions and break if found>>>
 305 | 	base = tokens[0]
 306 | 	suggestions = CommandSuggestions(base)
 307 | default:
 308 | 	<<<Check regex suggestions and break if found>>>
 309 | 	<<<Check file suggestions>>>
 310 | }
 311 | 
 312 | switch len(suggestions) {
 313 | case 0:
 314 | 	// Print BEL to warn that there were no suggestions.
 315 | 	fmt.Printf("\u0007")
 316 | case 1:
 317 | 	<<<Complete Suggestion>>>
 318 | default:
 319 | 	<<<Display Suggestions>>>
 320 | }
 321 | return nil
 322 | ```
 323 | 
 324 | 
 325 | At this point, there's no point in keeping it in the switch statement, because
 326 | it's in every case. Let's take it out:
 327 | 
 328 | ### "AutoCompletion Implementation"
 329 | ```go
 330 | tokens := c.Tokenize()
 331 | var suggestions []string
 332 | var base string
 333 | <<<Check regex suggestions>>>
 334 | if len(suggestions) == 0 {
 335 | 	switch len(tokens) {
 336 | 	case 0:
 337 | 		base = ""
 338 | 		suggestions = CommandSuggestions(base)
 339 | 	case 1:
 340 | 		base = tokens[0]
 341 | 		suggestions = CommandSuggestions(base)
 342 | 	default:
 343 | 		<<<Check file suggestions>>>
 344 | 	}
 345 | }
 346 | 
 347 | switch len(suggestions) {
 348 | case 0:
 349 | 	// Print BEL to warn that there were no suggestions.
 350 | 	fmt.Printf("\u0007")
 351 | case 1:
 352 | 	<<<Complete Suggestion>>>
 353 | default:
 354 | 	<<<Display Suggestions>>>
 355 | }
 356 | return nil
 357 | ```
 358 | 
 359 | We have still have a couple problems: ranging through a map is defined to be in
 360 | random order in Go. If we keep pressing tab with the autocomplete samples that
 361 | we used as our motivation, and type 'git show<tab>' we'll see that *sometimes*
 362 | it shows the rev-list, and sometimes it doesn't. Because of the "base" variable,
 363 | we probably don't have any choice but to keep two slices of suggestions: one for
 364 | "whole command" suggestions and one for "partial token" suggestions.
 365 | 
 366 | While we're at it, let's live dangerously and get rid of that indented switch
 367 | with a goto.
 368 | 
 369 | ### "AutoCompletion Implementation"
 370 | ```go
 371 | tokens := c.Tokenize()
 372 | var psuggestions, wsuggestions []string
 373 | var base string
 374 | 
 375 | <<<Check regex suggestions>>>
 376 | if len(wsuggestions) > 0 || len(psuggestions) > 0 {
 377 | 	goto foundSuggestions
 378 | }
 379 | 
 380 | switch len(tokens) {
 381 | case 0:
 382 | 	wsuggestions = CommandSuggestions(base)
 383 | case 1:
 384 | 	base = tokens[0]
 385 | 	psuggestions = CommandSuggestions(base)
 386 | default:
 387 | 	<<<Check file suggestions>>>
 388 | }
 389 | 
 390 | foundSuggestions:
 391 | switch len(psuggestions) + len(wsuggestions){
 392 | case 0:
 393 | 	// Print BEL to warn that there were no suggestions.
 394 | 	fmt.Printf("\u0007")
 395 | case 1:
 396 | 	if len(psuggestions) == 1 {
 397 | 		<<<Complete PSuggestion>>>
 398 | 	} else {
 399 | 		<<<Complete WSuggestion>>>
 400 | 	}
 401 | default:
 402 | 	<<<Display All Suggestions>>>
 403 | }
 404 | return nil
 405 | ```
 406 | 
 407 | We'll have to redefine some blocks based on previous blocks with new the new 
 408 | variable names (and without the break.)
 409 | 
 410 | ### "Check regex suggestions"
 411 | ```go
 412 | firstpart := strings.Join(tokens[:len(tokens)-1], " ")
 413 | wholecmd := strings.Join(tokens, " ")
 414 | base = tokens[len(tokens)-1]
 415 | for re, resuggestions := range autocompletions {
 416 | 	if re.MatchString(wholecmd) {
 417 | 		for _, val := range resuggestions {
 418 | 			// There was no last token, to take the prefix of, so
 419 | 			// just suggest the whole val.
 420 | 			wsuggestions = append(wsuggestions, string(val))
 421 | 		}
 422 | 	} else if re.MatchString(firstpart) {
 423 | 		for _, val := range resuggestions {
 424 | 			if string(val) != base && strings.HasPrefix(string(val), base) {
 425 | 				psuggestions = append(psuggestions, string(val))
 426 | 			}
 427 | 		}
 428 | 	}
 429 | }
 430 | ```
 431 | 
 432 | ### "Complete PSuggestion"
 433 | ```go
 434 | suggest := psuggestions[0]
 435 | *c = Command(strings.TrimSpace(string(*c)))
 436 | *c = Command(strings.TrimSuffix(string(*c), base))
 437 | *c += Command(suggest)
 438 | 
 439 | PrintPrompt()
 440 | fmt.Printf("%s", *c)
 441 | ```
 442 | 
 443 | ### "Complete WSuggestion"
 444 | ```go
 445 | suggest := wsuggestions[0]
 446 | *c = Command(strings.TrimSpace(string(*c)))
 447 | *c += Command(suggest)
 448 | 
 449 | PrintPrompt()
 450 | fmt.Printf("%s", *c)
 451 | ```
 452 | 
 453 | ### "Display All Suggestions"
 454 | ```go
 455 | fmt.Printf("\n%v\n", append(psuggestions, wsuggestions...))
 456 | 
 457 | PrintPrompt()
 458 | fmt.Printf("%s", *c)
 459 | ```
 460 | )
 461 | 
 462 | ### "Check file suggestions"
 463 | ```go
 464 | base = tokens[len(tokens)-1]
 465 | psuggestions = FileSuggestions(base)
 466 | ```
 467 | 
 468 | ## More flexible suggestions
 469 | 
 470 | We now have a basic customizable tab completion implementation, but what if
 471 | we want to make it a little more flexible?
 472 | 
 473 | Say we wanted `/^git show/` to suggest the output of the command
 474 | `git rev-list -n 10 HEAD` and suggest the last 10 commits? We can come up with
 475 | a convention like "if the suggestion token starts with a "!", then run the command
 476 | (without the "!"), and each line of the command run becomes a suggestion.
 477 | 
 478 | To do that, we'll start by adding a check for the first character inside of
 479 | our loop.
 480 | 
 481 | ### "Check regex suggestions"
 482 | ```go
 483 | firstpart := strings.Join(tokens[:len(tokens)-1], " ")
 484 | wholecmd := strings.Join(tokens, " ")
 485 | base = tokens[len(tokens)-1]
 486 | for re, resuggestions := range autocompletions {
 487 | 	if re.MatchString(wholecmd) {
 488 | 		for _, val := range resuggestions {
 489 | 			if len(val) > 2 && val[0] == '!' {
 490 | 				<<<WSuggest output of running command>>>
 491 | 			} else {
 492 | 				// There was no last token, to take the prefix of, so
 493 | 				// just suggest the whole val.
 494 | 				// As a special case, we still want to ignore it
 495 | 				// if the suggestion matches the last token, so
 496 | 				// that we don't step on psuggestion's feet.
 497 | 				if string(val) != base {
 498 | 					wsuggestions = append(wsuggestions, string(val))
 499 | 				}
 500 | 			}
 501 | 		}
 502 | 	} else if re.MatchString(firstpart) {
 503 | 		for _, val := range resuggestions {
 504 | 			// If it's length 1 it's just "!", and we should probably
 505 | 			// just suggest it literally.
 506 | 			if len(val) > 2 && val[0] == '!' {
 507 | 				<<<PSuggest output of running command>>>
 508 | 			} else if string(val) != base && strings.HasPrefix(string(val), base) {
 509 | 				psuggestions = append(psuggestions, string(val))
 510 | 			}
 511 | 		}
 512 | 	}
 513 | }
 514 | ```
 515 | 
 516 | Now, to get the output of the command we'll can just run os/exec.Output on val[1:],
 517 | because we don't need any of the fancy background/foreground semantics we needed
 518 | to handle for commands being executed by the user.
 519 | 
 520 | ### "PSuggest output of running command"
 521 | ```go
 522 | cmd := strings.Fields(string(val[1:]))
 523 | if len(cmd) < 1 {
 524 | 	continue
 525 | }
 526 | c := exec.Command(cmd[0], cmd[1:]...)
 527 | out, err := c.Output()
 528 | if err != nil {
 529 | 	println(err.Error())
 530 | 	continue
 531 | }
 532 | sugs := strings.Split(string(out), "\n")
 533 | for _, val := range sugs {
 534 | 	if val != base && strings.HasPrefix(val, base) {
 535 | 		psuggestions = append(psuggestions, val)
 536 | 	}
 537 | }
 538 | ```
 539 | 
 540 | ### "WSuggest output of running command"
 541 | ```go
 542 | cmd := strings.Fields(string(val[1:]))
 543 | if len(cmd) < 1 {
 544 | 	continue
 545 | }
 546 | c := exec.Command(cmd[0], cmd[1:]...)
 547 | out, err := c.Output()
 548 | if err != nil {
 549 | 	println(err.Error())
 550 | 	continue
 551 | }
 552 | sugs := strings.Split(string(out), "\n")
 553 | for _, val := range sugs {
 554 | 	if val != base {
 555 | 		wsuggestions = append(wsuggestions, val)
 556 | 	}
 557 | }
 558 | ```
 559 | 
 560 | 
 561 | ### "completion.go imports" +=
 562 | ```go
 563 | "os/exec"
 564 | ```
 565 | 
 566 | We now have much more flexible tab completion, but can we improve it even more?
 567 | 
 568 | Two ideas:
 569 | 1. Why don't we autocomplete partial matches?
 570 | 2. Why don't we make the regex subgroup matches available as a variable to the
 571 |    suggestions?
 572 | 
 573 | ## Partial Matching
 574 | 
 575 | Partial matching is fairly straight forward. We have a list of matches, and we
 576 | want to find the longest common prefix. There's probably a longest common
 577 | substring in Go that someone's written, but since we only care about the prefix
 578 | case we can do a naive implementation that ranges through our matches.
 579 | 
 580 | We'll start by adding a block to our AutoCompletion Implementation right before
 581 | Display Suggestions.
 582 | 
 583 | ### "AutoCompletion Implementation"
 584 | ```go
 585 | tokens := c.Tokenize()
 586 | var psuggestions, wsuggestions []string
 587 | var base string
 588 | 
 589 | <<<Check regex suggestions>>>
 590 | if len(wsuggestions) > 0 || len(psuggestions) > 0 {
 591 | 	goto foundSuggestions
 592 | }
 593 | 
 594 | switch len(tokens) {
 595 | case 0:
 596 | 	wsuggestions = CommandSuggestions(base)
 597 | case 1:
 598 | 	base = tokens[0]
 599 | 	psuggestions = CommandSuggestions(base)
 600 | default:
 601 | 	<<<Check file suggestions>>>
 602 | }
 603 | 
 604 | foundSuggestions:
 605 | switch len(psuggestions) + len(wsuggestions){
 606 | case 0:
 607 | 	// Print BEL to warn that there were no suggestions.
 608 | 	fmt.Printf("\u0007")
 609 | case 1:
 610 | 	if len(psuggestions) == 1 {
 611 | 		<<<Complete PSuggestion>>>
 612 | 	} else {
 613 | 		<<<Complete WSuggestion>>>
 614 | 	}
 615 | default:
 616 | 	<<<Complete Partial Matches>>>
 617 | 	<<<Display All Suggestions>>>
 618 | }
 619 | return nil
 620 | ```
 621 | 
 622 | our suggestions are stored in psuggestions and wsuggestions. We want the longest
 623 | prefix in the union of them. Let's start by creating a all of those. 
 624 | 
 625 | ### "Complete Partial Matches"
 626 | ```go
 627 | suggestions := append(psuggestions, wsuggestions...)
 628 | ```
 629 | 
 630 | Since we had to do something similar for displaying, let's update that code
 631 | to use our new union slice.
 632 | 
 633 | ### "Display All Suggestions"
 634 | ```go
 635 | fmt.Printf("\n%v\n", suggestions)
 636 | 
 637 | PrintPrompt()
 638 | fmt.Printf("%s", *c)
 639 | ```
 640 | 
 641 | 
 642 | Now, all we need to do is find the prefix. Let's start assume there's already a
 643 | LongestPrefix function, and add the completion code first, based on our
 644 | Psuggestion code:
 645 | 
 646 | ### "Complete Partial Matches" +=
 647 | ```go
 648 | suggest := LongestPrefix(suggestions)
 649 | *c = Command(strings.TrimSpace(string(*c)))
 650 | *c = Command(strings.TrimSuffix(string(*c), base))
 651 | *c += Command(suggest)
 652 | ```
 653 | 
 654 | Except that we've reintroduced the regression that caused us to split
 655 | wsuggestions from psuggestions in the first place. Let's ensure that 
 656 | 
 657 | ### "Complete Partial Matches"
 658 | ```go
 659 | ```
 660 | 
 661 | Then we can design with a table-driven test to make sure our LongestPrefix is
 662 | working as expected.
 663 | 
 664 | ### prefix_test.go
 665 | ```go
 666 | package main
 667 | 
 668 | import (
 669 | 	"testing"
 670 | 	<<<Prefix Test Imports>>>
 671 | )
 672 | 
 673 | func TestLongestPrefix(t *testing.T) {
 674 | 	cases := []struct{
 675 | 		Val []string
 676 | 		Expected string
 677 | 	}{
 678 | 		<<<LongestPrefix Test Cases>>>
 679 | 	}
 680 | 	for i, tc := range cases {
 681 | 		if got := LongestPrefix(tc.Val); got != tc.Expected {
 682 | 			t.Errorf("Unexpected prefix for case %d: got %v want %v", i, got, tc.Expected)
 683 | 		}
 684 | 	}
 685 | }
 686 | ```
 687 | 
 688 | And we should define our stub function:
 689 | 
 690 | ### prefix.go
 691 | ```go
 692 | package main
 693 | 
 694 | import (
 695 | 	<<<prefix.go imports>>>
 696 | )
 697 | 
 698 | func LongestPrefix(strs []string) string {
 699 | 	<<<LongestPrefix Implementation>>>
 700 | }
 701 | ```
 702 | 
 703 | And stubs for the blocks we just defined so that it will compile.
 704 | ### "prefix.go imports"
 705 | ```go
 706 | ```
 707 | 
 708 | ### "Prefix Test Imports"
 709 | ```go
 710 | ```
 711 | 
 712 | ### "LongestPrefix Implementation"
 713 | ```go
 714 | return ""
 715 | ```
 716 | 
 717 | 
 718 | Now, we'll define some real test cases.
 719 | 
 720 | ### "LongestPrefix Test Cases"
 721 | ```go
 722 | // Empty case or nil slice
 723 | { nil, ""},
 724 | { []string{}, "" },
 725 | 
 726 | // Prefix of 1 element is itself
 727 | { []string{"a"}, "a"},
 728 | 
 729 | // 2 elements with no prefix
 730 | { []string{"a", "b"}, "" },
 731 | 
 732 | // 2 elements with a common prefix
 733 | { []string{"aa", "ab"}, "a"},
 734 | 
 735 | // multiple elements
 736 | { []string{"aaaa", "aabb", "aaac"}, "aa"},
 737 | ```
 738 | 
 739 | And as expected, our tests fail.
 740 | 
 741 | So for our implementation, let's start by assuming that strs[0] is the longest
 742 | prefix, then we'll go through the remaining elements, range through them, and
 743 | if anything doesn't match, take the smaller prefix and go on to the next element.
 744 | 
 745 | (We'll also add a quick check for the empty case to make sure it doesn't crash.)
 746 | 
 747 | ### "LongestPrefix Implementation"
 748 | ```go
 749 | if len(strs) == 0 {
 750 | 	return ""
 751 | }
 752 | 
 753 | prefix := strs[0]
 754 | for _, cmp := range strs[1:] {
 755 | 	for i := range prefix {
 756 | 		if i > len(cmp) || prefix[i] != cmp[i]{
 757 | 			prefix = cmp[:i]
 758 | 			break
 759 | 		}
 760 | 	}
 761 | }
 762 | return prefix
 763 | ```
 764 | 
 765 | And now our partial completions work, except we're reintroduced the regression
 766 | which caused us to split psuggestions and wsuggestions in the first place. The
 767 | easiest way to handle this is probably do only do prefix suggestions if there
 768 | are no whole-command based suggestions (wsuggestions), because if the user hasn't
 769 | even started typing one letter of the last token that we're matching, the prefix
 770 | completion isn't very useful anyways. (We'll keep the definition of "suggestions"
 771 | since it's referenced in the display block, even if it's meaningless for prefix
 772 | completions now.)
 773 | 
 774 | ### "Complete Partial Matches"
 775 | ```go
 776 | suggestions := append(psuggestions, wsuggestions...)
 777 | 
 778 | if len(wsuggestions) == 0 {
 779 | 	suggest := LongestPrefix(suggestions)
 780 | 	*c = Command(strings.TrimSpace(string(*c)))
 781 | 	*c = Command(strings.TrimSuffix(string(*c), base))
 782 | 	*c += Command(suggest)
 783 | }
 784 | ```
 785 | 
 786 | ## Subgroup Regexp Matching
 787 | 
 788 | Subgroup matching shouldn't be difficult to add, because any regexp engine
 789 | already implements it. Let's start by replacing MatchString with
 790 | FindStringSubmatch when checking our regular expressions, so that we have the
 791 | values that we're going to need later.
 792 | 
 793 | ### "Check regex suggestions"
 794 | ```go
 795 | var firstpart string
 796 | if len(tokens) > 0 {
 797 | 	base = tokens[len(tokens)-1]
 798 | 	firstpart = strings.Join(tokens[:len(tokens)-1], " ")
 799 | }
 800 | wholecmd := strings.Join(tokens, " ")
 801 | for re, resuggestions := range autocompletions {
 802 | 	if matches := re.FindStringSubmatch(wholecmd); matches != nil {
 803 | 		for _, val := range resuggestions {
 804 | 			if len(val) > 2 && val[0] == '!' {
 805 | 				<<<WSuggest output of running command>>>
 806 | 			} else {
 807 | 				// There was no last token, to take the prefix of, so
 808 | 				// just suggest the whole val.
 809 | 				// As a special case, we still want to ignore it
 810 | 				// if the suggestion matches the last token, so
 811 | 				// that we don't step on psuggestion's feet.
 812 | 				if string(val) != base {
 813 | 					wsuggestions = append(wsuggestions, string(val))
 814 | 				}
 815 | 			}
 816 | 		}
 817 | 	} else if matches := re.FindStringSubmatch(firstpart); matches != nil {
 818 | 		for _, val := range resuggestions {
 819 | 			// If it's length 1 it's just "!", and we should probably
 820 | 			// just suggest it literally.
 821 | 			if len(val) > 2 && val[0] == '!' {
 822 | 				<<<PSuggest output of running command>>>
 823 | 			} else if string(val) != base && strings.HasPrefix(string(val), base) {
 824 | 				psuggestions = append(psuggestions, string(val))
 825 | 			}
 826 | 		}
 827 | 	}
 828 | }
 829 | ```
 830 | 
 831 | Now, we should just need to replace \n in the string with matches[n] in our
 832 | suggestions. The `os` package has an `os.Expand` helper that can do this for $n,
 833 | but if we tried to use "$" to represent the submatches, we'd have trouble
 834 | interpretting the "autocomplete" builtin when defining our completions, because
 835 | the $n would be expanded to an environment variable before it got to us.
 836 | 
 837 | Luckily, it's not hard to just use a loop that replaces \n with matches[n]
 838 | blindly.
 839 | 
 840 | ### "Check regex suggestions"
 841 | ```go
 842 | var firstpart string
 843 | if len(tokens) > 0 {
 844 | 	base = tokens[len(tokens)-1]
 845 | 	firstpart = strings.Join(tokens[:len(tokens)-1], " ")
 846 | }
 847 | wholecmd := strings.Join(tokens, " ")
 848 | for re, resuggestions := range autocompletions {
 849 | 	if matches := re.FindStringSubmatch(wholecmd); matches != nil {
 850 | 		for _, val := range resuggestions {
 851 | 			<<<Expand Matches>>>
 852 | 
 853 | 			if len(val) > 2 && val[0] == '!' {
 854 | 				<<<WSuggest output of running command>>>
 855 | 			} else {
 856 | 				// There was no last token, to take the prefix of, so
 857 | 				// just suggest the whole val.
 858 | 				// As a special case, we still want to ignore it
 859 | 				// if the suggestion matches the last token, so
 860 | 				// that we don't step on psuggestion's feet.
 861 | 				if string(val) != base {
 862 | 					wsuggestions = append(wsuggestions, string(val))
 863 | 				}
 864 | 			}
 865 | 		}
 866 | 	} else if matches := re.FindStringSubmatch(firstpart); matches != nil {
 867 | 		for _, val := range resuggestions {
 868 | 			<<<Expand Matches>>>
 869 | 
 870 | 			// If it's length 1 it's just "!", and we should probably
 871 | 			// just suggest it literally.
 872 | 			if len(val) > 2 && val[0] == '!' {
 873 | 				<<<PSuggest output of running command>>>
 874 | 			} else if string(val) != base && strings.HasPrefix(string(val), base) {
 875 | 				psuggestions = append(psuggestions, string(val))
 876 | 			}
 877 | 		}
 878 | 	}
 879 | }
 880 | ```
 881 | 
 882 | ### "Expand Matches"
 883 | ```go
 884 | for n, match := range matches {
 885 | 	val = Token(strings.Replace(string(val), fmt.Sprintf(`\%d`, n), match, -1))
 886 | }
 887 | ```
 888 | 
 889 | This is going to be horribly inefficient as the number of substring matches
 890 | increases in a regex, but shouldn't affect the general case where people aren't
 891 | using submatches much, so there isn't much need to optimize it right now. The
 892 | most powerful use of the substring matches is probably to use it to pass
 893 | arguments to an autocomplete suggester program, and the overhead of invoking a
 894 | separate program in that case is probably going to dwarf our loop anyways.
 895 | 
 896 | Now, \0 (or any submatch) may have a space in it, which makes the display confusing.
 897 | Let's update our printing to include quotation marks if the suggestion has a
 898 | space or a tab.
 899 | 
 900 | ### "Display All Suggestions"
 901 | ```go
 902 | fmt.Printf("\n[")
 903 | for  i, s := range suggestions {
 904 | 	if strings.ContainsAny(s, " \t") {
 905 | 		fmt.Printf(`"%v"`, s)
 906 | 	} else {
 907 | 		fmt.Printf("%v", s)
 908 | 	}
 909 | 	if i != len(suggestions)-1 {
 910 | 		fmt.Printf(" ")
 911 | 	} 
 912 | }
 913 | fmt.Printf("]\n")
 914 | 
 915 | PrintPrompt()
 916 | fmt.Printf("%s", *c)
 917 | ```
 918 | 
 919 | We still seem to have a bug where, if regexes match both psuggestions and
 920 | wsuggestions, the wsuggestions will always get suggested even if they don't
 921 | match the partially typed string. To fix that, let's just set wsuggestions to
 922 | nil if there are any psuggestions to make sure they get priority.
 923 | 
 924 | ### "AutoCompletion Implementation"
 925 | ```go
 926 | tokens := c.Tokenize()
 927 | var psuggestions, wsuggestions []string
 928 | var base string
 929 | 
 930 | <<<Check regex suggestions>>>
 931 | if len(psuggestions) > 0 {
 932 | 	wsuggestions = nil
 933 | 	goto foundSuggestions
 934 | } else if len(wsuggestions) > 0 {
 935 | 	goto foundSuggestions
 936 | }
 937 | 
 938 | switch len(tokens) {
 939 | case 0:
 940 | 	base = ""
 941 | 	wsuggestions = CommandSuggestions(base)
 942 | case 1:
 943 | 	base = tokens[0]
 944 | 	psuggestions = CommandSuggestions(base)
 945 | default:
 946 | 	<<<Check file suggestions>>>
 947 | }
 948 | 
 949 | foundSuggestions:
 950 | switch len(psuggestions) + len(wsuggestions){
 951 | case 0:
 952 | 	// Print BEL to warn that there were no suggestions.
 953 | 	fmt.Printf("\u0007")
 954 | case 1:
 955 | 	if len(psuggestions) == 1 {
 956 | 		<<<Complete PSuggestion>>>
 957 | 	} else {
 958 | 		<<<Complete WSuggestion>>>
 959 | 	}
 960 | default:
 961 | 	<<<Complete Partial Matches>>>
 962 | 	<<<Display All Suggestions>>>
 963 | }
 964 | return nil
 965 | ```
 966 | 
 967 | That still didn't do it because we're doing something silly and psuggestions
 968 | in an else if block after wsuggestions, so let's fix that. Now that we have the
 969 | psuggestion always takes precedence over wsuggestions logic, we can just skip
 970 | checking wsuggestions if there's any psuggestions found.
 971 | 
 972 | ### "Check regex suggestions"
 973 | ```go
 974 | var firstpart string
 975 | if len(tokens) > 0 {
 976 | 	base = tokens[len(tokens)-1]
 977 | 	firstpart = strings.Join(tokens[:len(tokens)-1], " ")
 978 | }
 979 | wholecmd := strings.Join(tokens, " ")
 980 | 
 981 | for re, resuggestions := range autocompletions {
 982 | 	if matches := re.FindStringSubmatch(firstpart); matches != nil {
 983 | 		for _, val := range resuggestions {
 984 | 			<<<Expand Matches>>>
 985 | 
 986 | 			// If it's length 1 it's just "!", and we should probably
 987 | 			// just suggest it literally.
 988 | 			if len(val) > 2 && val[0] == '!' {
 989 | 				<<<PSuggest output of running command>>>
 990 | 			} else if string(val) != base && strings.HasPrefix(string(val), base) {
 991 | 				psuggestions = append(psuggestions, string(val))
 992 | 			}
 993 | 		}
 994 | 	}
 995 | 
 996 | 	if len(psuggestions) > 0 {
 997 | 		continue
 998 | 	}
 999 | 
1000 | 	if matches := re.FindStringSubmatch(wholecmd); matches != nil {
1001 | 		for _, val := range resuggestions {
1002 | 			<<<Expand Matches>>>
1003 | 
1004 | 			if len(val) > 2 && val[0] == '!' {
1005 | 				<<<WSuggest output of running command>>>
1006 | 			} else {
1007 | 				// There was no last token, to take the prefix of, so
1008 | 				// just suggest the whole val.
1009 | 				wsuggestions = append(wsuggestions, string(val))
1010 | 			}
1011 | 		}
1012 | 	}
1013 | }
1014 | ```
1015 | 
1016 | ## Conclusions
1017 | 
1018 | We now have an `autocomplete` builtin which can suggest autocompletions for user
1019 | input based on comparing what they've typed to a regular expression. The
1020 | suggestions can either be simple strings, based on (sub) matches of the regex, 
1021 | or based on invoking a separate program (potentially with arguments, which may
1022 | or may not come from the regex).
1023 | 
1024 | We have the potential for great, customizable tab completion, we just need
1025 | some examples.
1026 | 
1027 | I've included a sample goshrc in this repo with some simple examples that I use
1028 | in my `~/.goshrc` file. Feel free to send pull requests to it with anything else
1029 | you find useful, and be as creative as you'd like.
1030 | 


--------------------------------------------------------------------------------
/Tokenization.md:
--------------------------------------------------------------------------------
  1 | # Input Tokenization
  2 | 
  3 | At the end of our basic shell, we said we needed better tokenization
  4 | support. Recall, we're currently just calling `strings.Fields(cmd)` to
  5 | split the command into strings on whitespace.
  6 | 
  7 | This means that if we enter, for instance, `git commit -m 'I am a message'`,
  8 | it gets split into the slice `[]string{"git", "commit", "-m", "'I", "am", "a",
  9 | "message'"}`, when what we probably intended was `[]string{"git", "commit",
 10 | "-m", "I am a message"}`
 11 | 
 12 | Splitting on whitespace was easy, because there was no thinking or design to
 13 | do. More complex tokenization is.. more complex. Do we want a POSIX compliant
 14 | sh shell? Do we want something more similar to csh syntax? Do we want to go with
 15 | a syntax more similar to Plan 9's rc shell? Or do we want to come up with
 16 | something on our own?
 17 | 
 18 | Let's start with something simple. We'll just add the ability to use `'` to
 19 | declare a string literal until the next time we see a `'` (Unless there's a
 20 | `\` before the `'`, in which case we'll escape it and include a `'` in the
 21 | literal.) We'll also want to add pipeline support eventually, so if we see a `|`
 22 | outside of a string literal, we'll use that as a delimiter too.
 23 | 
 24 | We should be able to do all this fairly easily on a single pass of cmd.
 25 | 
 26 | ## Tests
 27 | 
 28 | I hate parsing, and I'm never confident I'm doing it properly (I'm usually not),
 29 | so we'll start with some table driven tests of the basic use cases.
 30 | 
 31 | ### tokenize_test.go
 32 | ```go
 33 | package main
 34 | 
 35 | import (
 36 | 	"testing"
 37 | 	<<<other tokenize_test.go imports>>>
 38 | )
 39 | 
 40 | func TestTokenization(t *testing.T) {
 41 | 	tests := []struct {
 42 | 		cmd      Command
 43 | 		expected []string
 44 | 	}{
 45 | 		{cmd: "ls", expected: []string{"ls"}},
 46 | 		{"     ls    	", []string{"ls"}},
 47 | 		{"ls -l", []string{"ls", "-l"}},
 48 | 		{"git commit -m 'I am message'", []string{"git", "commit", "-m", "I am message"}},
 49 | 		{"git commit -m 'I\\'m another message'", []string{"git", "commit", "-m", "I'm another message"}},
 50 | 		{"ls|cat", []string{"ls", "|", "cat"}},
 51 | 	}
 52 | 	for i, tc := range tests {
 53 | 		val := tc.cmd.Tokenize()
 54 | 		if len(val) != len(tc.expected) {
 55 | 			// The below loop might panic if the lengths aren't equal, so this is fatal instead of an error.
 56 | 			t.Fatalf("Mismatch for result length in test case %d. Got '%v' want '%v'", i, len(val), len(tc.expected))
 57 | 		}
 58 | 		for j, token := range val {
 59 | 			if token != tc.expected[j] {
 60 | 				t.Errorf("Mismatch for index %d in test case %d. Got '%v' want '%v'", j, i, token, tc.expected[j])
 61 | 			}
 62 | 		}
 63 | 	}
 64 | }
 65 | ```
 66 | 
 67 | We also need to define Command.Tokenize(). We'll start with a little refactoring
 68 | for an implementation that just uses `strings.Fields` (and expect it to fail)
 69 | the tests that we just wrote.
 70 | 
 71 | ### "HandleCmd Implementation"
 72 | ```go
 73 | func (c Command) HandleCmd() error {
 74 | 	parsed := c.Tokenize()
 75 | 	if len(parsed) == 0 {
 76 | 		// There was no command, it's not an error, the user just hit
 77 | 		// enter.
 78 | 		PrintPrompt()
 79 | 		return nil
 80 | 	}
 81 | 
 82 | 	var args []string
 83 | 	for _, val := range parsed[1:] {
 84 | 		if val[0] == '$' {
 85 | 			args = append(args, os.Getenv(val[1:]))
 86 | 		} else {
 87 | 			args = append(args, val)
 88 | 		}
 89 | 	}
 90 | 
 91 | 	if parsed[0] == "cd" {
 92 | 		if len(args) == 0 {
 93 | 			return fmt.Errorf("Must provide an argument to cd")
 94 | 		}
 95 | 		return os.Chdir(args[0])
 96 | 
 97 | 	}
 98 | 
 99 | 	cmd := exec.Command(parsed[0], args...)
100 | 	cmd.Stdin = os.Stdin
101 | 	cmd.Stdout = os.Stdout
102 | 	cmd.Stderr = os.Stderr
103 | 
104 | 	return cmd.Run()
105 | }
106 | ```
107 | 
108 | (It looks like we did a lot there, but it's just the implementation
109 | from the end of the README.md with `parsed := strings.Fields(string(c))`
110 | replaced with `parsed := c.Tokenize()` on the first line.)
111 | 
112 | We'll start moving our new code into another file to keep it cleaner,
113 | which is going to have a similar structure to main.go.
114 | 
115 | ### tokenize.go
116 | ```go
117 | package main
118 | 
119 | import (
120 | 	<<<tokenize.go imports>>>
121 | )
122 | 
123 | <<<tokenize.go globals>>>
124 | ```
125 | 
126 | ### "tokenize.go globals"
127 | ```
128 | func (c Command) Tokenize() []string {
129 | 	<<<Tokenize Implementation>>>
130 | }
131 | ```
132 | 
133 | Finally, the implementation.
134 | 
135 | ### "Tokenize Implementation"
136 | ```go
137 | return strings.Fields(string(c))
138 | ```
139 | 
140 | ### "tokenize.go imports" +=
141 | ```go
142 | "strings"
143 | ```
144 | 
145 | Some refactoring overhead: we'll have to remove "strings" from our 
146 | main.go imports, otherwise Go will refuse to compile. Our list ends
147 | up being:
148 | 
149 | ### "main.go imports"
150 | ```go
151 | "bufio"
152 | "fmt"
153 | "github.com/pkg/term"
154 | "os"
155 | "os/exec"
156 | ```
157 | 
158 | And now if we run our test...
159 | 
160 | ```
161 | > go test ./...
162 | --- FAIL: TestTokenization (0.00s)
163 | 	tokenize_test.go:22: Mismatch for result length in test case 3. Got 6 want 4
164 | FAIL
165 | FAIL	github.com/driusan/gosh	0.006s
166 | exit status 1
167 | ```
168 | 
169 | Hooray! It failed on the 4th test case as expected! (We just printed the index
170 | in the error, which is zero indexed.)
171 | 
172 | Now that we've done our refactoring and written our tests, we can worry about
173 | our implementation.
174 | 
175 | ## Tokenization, for real.
176 | 
177 | So how are we going to actually implement this? We'll need to iterate
178 | through our command string, and keep track of some things every time we see
179 | a `'`. Are we in a string? If so, this is the start of one. If not, it's the
180 | end of one unless the previous character is a `\`. If we're in a string, we 
181 | don't care about anything other than where the end of the string is. If we're
182 | not, we care about: is this a whitespace character? If so, we want to ignore it,
183 | (or append the previous token to the parsed arguments if the previous character
184 | wasn't whitespace.)
185 | 
186 | We can use `unicode.IsWhitespace()` for the whitespace checking, and a token
187 | start int to keep track of where the start of the current token is so that
188 | we can append to the parsed args slice when we get to the end of one.
189 | 
190 | ### "Tokenize Implementation"
191 | ```go
192 | var parsed []string
193 | tokenStart := -1
194 | inStringLiteral := false
195 | for i, chr := range c {
196 | 	switch chr {
197 | 		case '\'':
198 | 			<<<Handle Quote>>>
199 | 		default:
200 | 			<<<Handle Nonquote>>>
201 | 	}
202 | }
203 | return parsed
204 | ```
205 | 
206 | We'll start with the handling of the character `'`. We need to know
207 | if it's the start or end of our string literal. If it's the start,
208 | mark it, and if it's the end, add it to the parsed tokens.
209 | 
210 | ### "Handle Quote"
211 | ```go
212 | if inStringLiteral {
213 | 	if i > 0 && c[i-1] == '\\' {
214 | 		// The quote was escaped, so ignore it.
215 | 		continue
216 | 	}
217 | 	inStringLiteral = false
218 | 
219 | 	token := string(c[tokenStart:i])
220 | 
221 | 	// Replace escaped quotes with just a single ' before appending
222 | 	token = strings.Replace(token, `\'`, "'", -1)
223 | 	parsed = append(parsed, token)
224 | 
225 | 	// Now that we've finished, reset the tokenStart for the next token.
226 | 	tokenStart = -1
227 | } else {
228 | 	// This is the quote, which means the literal starts at the next
229 | 	// character
230 | 	tokenStart = i+1
231 | 	inStringLiteral = true
232 | }
233 | ```
234 | 
235 | Now, for handling non-quotation characters. If we're in the middle of a string
236 | literal, we just want to ignore it and let it be taken care of above. Otherwise,
237 | if it's a whitespace, we end the current token and add it it to the parsed
238 | arguments. if it's a whitespace character, we've either reached the end of a 
239 | token and should add it to the parsed arguments, or we're not in a token and
240 | should just ignore it.
241 | 
242 | ### "Handle Nonquote"
243 | ```go
244 | if inStringLiteral {
245 | 	continue
246 | }
247 | if unicode.IsSpace(chr) {
248 | 	if tokenStart == -1 {
249 | 		continue
250 | 	}
251 | 	parsed = append(parsed, string(c[tokenStart:i]))
252 | 	tokenStart = -1
253 | } else if tokenStart == -1 {
254 | 	tokenStart = i
255 | }
256 | ```
257 | 
258 | ### "tokenize.go imports" +=
259 | ```go
260 | "unicode"
261 | ```
262 | 
263 | And when we run our tests... 
264 | 
265 | ```
266 | > go test ./... 
267 | --- FAIL: TestTokenization (0.00s)
268 | 	tokenize_test.go:22: Mismatch for result length in test case 0. Got 0 want 1
269 | FAIL
270 | FAIL	github.com/driusan/gosh	0.005s
271 | 
272 | ```
273 | 
274 | It now fails on the first test. I told you I always got parsing wrong, so it's
275 | a good thing we wrote those tets. It got 0 results and expected 1, suggesting
276 | we're doing something wrong with the last (or first) token.
277 | 
278 | In fact, there's no whitespace at the end of the last token, and we forgot to
279 | take care of that. So after the loop, let's check if tokenStart is >= 0 and
280 | add the final token if so.
281 | 
282 | ### "Tokenize Implementation"
283 | ```go
284 | var parsed []string
285 | tokenStart := -1
286 | inStringLiteral := false
287 | for i, chr := range c {
288 | 	switch chr {
289 | 		case '\'':
290 | 			<<<Handle Quote>>>
291 | 		default:
292 | 			<<<Handle Nonquote>>>
293 | 	}
294 | }
295 | if tokenStart >= 0 {
296 | 	if inStringLiteral {
297 | 		// Ignore the ' character
298 | 		tokenStart += 1
299 | 	}
300 | 	parsed = append(parsed, string(c[tokenStart:]))
301 | }
302 | return parsed
303 | ```
304 | 
305 | Now when we run `go test ./...` we get to the last `ls|cat` test, which isn't
306 | surprising since we didn't implement '|' as a delimiter.
307 | 
308 | ```
309 | > go test            
310 | --- FAIL: TestTokenization (0.00s)
311 | 	tokenize_test.go:22: Mismatch for result length in test case 4. Got '1' want '3'
312 | FAIL
313 | exit status 1
314 | FAIL	github.com/driusan/gosh	0.005s
315 | ```
316 | 
317 | Let's add it to our switch statement, and do a little refactoring of our
318 | of our Tokenize implementation into smaller semantic chunks while we're at it. 
319 | 
320 | ### "Tokenize Implementation"
321 | ```go
322 | <<<Tokenize Globals>>>
323 | for i, chr := range c {
324 | 	<<<Handle Tokenize Chr>>>
325 | }
326 | <<<Add Last Token>>>
327 | ```
328 | 
329 | ### "Tokenize Globals"
330 | ```go
331 | var parsed []string
332 | tokenStart := -1
333 | inStringLiteral := false
334 | ```
335 | 
336 | ### "Handle Tokenize Chr"
337 | ```go
338 | switch chr {
339 | 	case '\'':
340 | 		<<<Handle Quote>>>
341 | 	case '|':
342 | 		<<<Handle Pipe Chr>>>
343 | 	default:
344 | 		<<<Handle Nonquote>>>
345 | }
346 | ```
347 | 
348 | ### "Add Last Token"
349 | ```go
350 | if tokenStart >= 0 {
351 | 	if inStringLiteral {
352 | 		// Ignore the ' character
353 | 		tokenStart += 1
354 | 	}
355 | 	parsed = append(parsed, string(c[tokenStart:]))
356 | }
357 | return parsed
358 | ```
359 | 
360 | The logic of Handle Pipe Chr is going to be pretty straight forward. If we're
361 | in a string literal, don't. Otherwise, add any existing token and add it
362 | as well as a `|` token.
363 | 
364 | ### "Handle Pipe Chr"
365 | ```go
366 | if inStringLiteral {
367 | 	continue
368 | }
369 | 
370 | if tokenStart >= 0 {
371 | 	parsed = append(parsed, string(c[tokenStart:i]))
372 | } else {
373 | 	parsed = append(parsed, string(c[:i]))
374 | }
375 | parsed = append(parsed, "|")
376 | tokenStart = -1
377 | ```
378 | 
379 | And now...
380 | 
381 | ```
382 | > go test
383 | PASS
384 | ok  	github.com/driusan/gosh	0.005s
385 | ```
386 | 
387 | We can pass arguments to programs with strings!
388 | 
389 | There's probably better ways to do this (like using the
390 | [`text/scanner`](https://golang.org/pkg/text/scanner/) package and
391 | we may want to revisit later, but for now this works.
392 | 
393 | We also never used the `other tokenize_test.go imports` macro, so let's define
394 | an empty one to avoid compilation errors:
395 | 
396 | ### "other tokenize_test.go imports"
397 | ```go
398 | ```
399 | 
400 | 


--------------------------------------------------------------------------------
/completion.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"fmt"
  5 | 	"io/ioutil"
  6 | 	"os"
  7 | 	"os/exec"
  8 | 	"path/filepath"
  9 | 	"regexp"
 10 | 	"strings"
 11 | )
 12 | 
 13 | var autocompletions map[*regexp.Regexp][]Token
 14 | 
 15 | func (c *Command) Complete() error {
 16 | 	tokens := c.Tokenize()
 17 | 	var psuggestions, wsuggestions []string
 18 | 	var base string
 19 | 
 20 | 	var firstpart string
 21 | 	if len(tokens) > 0 {
 22 | 		base = tokens[len(tokens)-1]
 23 | 		firstpart = strings.Join(tokens[:len(tokens)-1], " ")
 24 | 	}
 25 | 	wholecmd := strings.Join(tokens, " ")
 26 | 
 27 | 	for re, resuggestions := range autocompletions {
 28 | 		if matches := re.FindStringSubmatch(firstpart); matches != nil {
 29 | 			for _, val := range resuggestions {
 30 | 				for n, match := range matches {
 31 | 					val = Token(strings.Replace(string(val), fmt.Sprintf(`\%d`, n), match, -1))
 32 | 				}
 33 | 
 34 | 				// If it's length 1 it's just "!", and we should probably
 35 | 				// just suggest it literally.
 36 | 				if len(val) > 2 && val[0] == '!' {
 37 | 					cmd := strings.Fields(string(val[1:]))
 38 | 					if len(cmd) < 1 {
 39 | 						continue
 40 | 					}
 41 | 					c := exec.Command(cmd[0], cmd[1:]...)
 42 | 					out, err := c.Output()
 43 | 					if err != nil {
 44 | 						println(err.Error())
 45 | 						continue
 46 | 					}
 47 | 					sugs := strings.Split(string(out), "\n")
 48 | 					for _, val := range sugs {
 49 | 						if val != base && strings.HasPrefix(val, base) {
 50 | 							psuggestions = append(psuggestions, val)
 51 | 						}
 52 | 					}
 53 | 				} else if string(val) != base && strings.HasPrefix(string(val), base) {
 54 | 					psuggestions = append(psuggestions, string(val))
 55 | 				}
 56 | 			}
 57 | 		}
 58 | 
 59 | 		if len(psuggestions) > 0 {
 60 | 			continue
 61 | 		}
 62 | 
 63 | 		if matches := re.FindStringSubmatch(wholecmd); matches != nil {
 64 | 			for _, val := range resuggestions {
 65 | 				for n, match := range matches {
 66 | 					val = Token(strings.Replace(string(val), fmt.Sprintf(`\%d`, n), match, -1))
 67 | 				}
 68 | 
 69 | 				if len(val) > 2 && val[0] == '!' {
 70 | 					cmd := strings.Fields(string(val[1:]))
 71 | 					if len(cmd) < 1 {
 72 | 						continue
 73 | 					}
 74 | 					c := exec.Command(cmd[0], cmd[1:]...)
 75 | 					out, err := c.Output()
 76 | 					if err != nil {
 77 | 						println(err.Error())
 78 | 						continue
 79 | 					}
 80 | 					sugs := strings.Split(string(out), "\n")
 81 | 					for _, val := range sugs {
 82 | 						if val != base {
 83 | 							wsuggestions = append(wsuggestions, val)
 84 | 						}
 85 | 					}
 86 | 				} else {
 87 | 					// There was no last token, to take the prefix of, so
 88 | 					// just suggest the whole val.
 89 | 					wsuggestions = append(wsuggestions, string(val))
 90 | 				}
 91 | 			}
 92 | 		}
 93 | 	}
 94 | 	if len(psuggestions) > 0 {
 95 | 		wsuggestions = nil
 96 | 		goto foundSuggestions
 97 | 	} else if len(wsuggestions) > 0 {
 98 | 		goto foundSuggestions
 99 | 	}
100 | 
101 | 	switch len(tokens) {
102 | 	case 0:
103 | 		base = ""
104 | 		wsuggestions = CommandSuggestions(base)
105 | 	case 1:
106 | 		base = tokens[0]
107 | 		psuggestions = CommandSuggestions(base)
108 | 	default:
109 | 		base = tokens[len(tokens)-1]
110 | 		psuggestions = FileSuggestions(base)
111 | 	}
112 | 
113 | foundSuggestions:
114 | 	switch len(psuggestions) + len(wsuggestions) {
115 | 	case 0:
116 | 		// Print BEL to warn that there were no suggestions.
117 | 		fmt.Printf("\u0007")
118 | 	case 1:
119 | 		if len(psuggestions) == 1 {
120 | 			suggest := psuggestions[0]
121 | 			*c = Command(strings.TrimSpace(string(*c)))
122 | 			*c = Command(strings.TrimSuffix(string(*c), base))
123 | 			*c += Command(suggest)
124 | 
125 | 			PrintPrompt()
126 | 			fmt.Printf("%s", *c)
127 | 		} else {
128 | 			suggest := wsuggestions[0]
129 | 			*c = Command(strings.TrimSpace(string(*c)))
130 | 			*c += Command(suggest)
131 | 
132 | 			PrintPrompt()
133 | 			fmt.Printf("%s", *c)
134 | 		}
135 | 	default:
136 | 		suggestions := append(psuggestions, wsuggestions...)
137 | 
138 | 		if len(wsuggestions) == 0 {
139 | 			suggest := LongestPrefix(suggestions)
140 | 			*c = Command(strings.TrimSpace(string(*c)))
141 | 			*c = Command(strings.TrimSuffix(string(*c), base))
142 | 			*c += Command(suggest)
143 | 		}
144 | 		fmt.Printf("\n[")
145 | 		for i, s := range suggestions {
146 | 			if strings.ContainsAny(s, " \t") {
147 | 				fmt.Printf(`"%v"`, s)
148 | 			} else {
149 | 				fmt.Printf("%v", s)
150 | 			}
151 | 			if i != len(suggestions)-1 {
152 | 				fmt.Printf(" ")
153 | 			}
154 | 		}
155 | 		fmt.Printf("]\n")
156 | 
157 | 		PrintPrompt()
158 | 		fmt.Printf("%s", *c)
159 | 	}
160 | 	return nil
161 | }
162 | 
163 | func CommandSuggestions(base string) []string {
164 | 	paths := strings.Split(os.Getenv("PATH"), ":")
165 | 	var matches []string
166 | 	for _, path := range paths {
167 | 		// We don't care if there's an invalid path in $PATH, so ignore
168 | 		// the error.
169 | 		files, _ := ioutil.ReadDir(path)
170 | 		for _, file := range files {
171 | 			if name := file.Name(); strings.HasPrefix(name, base) {
172 | 				matches = append(matches, name)
173 | 			}
174 | 		}
175 | 	}
176 | 	return matches
177 | }
178 | 
179 | func FileSuggestions(base string) []string {
180 | 	base = replaceTilde(base)
181 | 	if files, err := ioutil.ReadDir(base); err == nil {
182 | 		// This was a directory, so use the empty string as a prefix.
183 | 		fileprefix := ""
184 | 		filedir := base
185 | 		var matches []string
186 | 		for _, file := range files {
187 | 			if name := file.Name(); strings.HasPrefix(name, fileprefix) {
188 | 				matches = append(matches, filepath.Clean(filedir+"/"+name))
189 | 			}
190 | 		}
191 | 		return matches
192 | 	}
193 | 
194 | 	filedir := filepath.Dir(base)
195 | 	fileprefix := filepath.Base(base)
196 | 	files, err := ioutil.ReadDir(filedir)
197 | 	if err != nil {
198 | 		return nil
199 | 	}
200 | 
201 | 	var matches []string
202 | 	for _, file := range files {
203 | 		if name := file.Name(); strings.HasPrefix(name, fileprefix) {
204 | 			matches = append(matches, filepath.Clean(filedir+"/"+name))
205 | 		}
206 | 	}
207 | 	return matches
208 | }
209 | 


--------------------------------------------------------------------------------
/goshrc:
--------------------------------------------------------------------------------
1 | autocomplete ^git$ add commit checkout show 'rebase -i'
2 | autocomplete '^git show$' '!git rev-list -n 10 HEAD'
3 | autocomplete '^git add' '!git ls-files --exclude-standard -m -o -d'
4 | autocomplete '^git commit$' '--amend' '-m'
5 | 


--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"bufio"
  5 | 	"errors"
  6 | 	"fmt"
  7 | 	"github.com/pkg/term"
  8 | 	"io"
  9 | 	"os"
 10 | 	"os/exec"
 11 | 	"os/signal"
 12 | 	"os/user"
 13 | 	"path/filepath"
 14 | 	"regexp"
 15 | 	"strconv"
 16 | 	"strings"
 17 | 	"syscall"
 18 | 	"unsafe"
 19 | )
 20 | 
 21 | type Command string
 22 | type ParsedCommand struct {
 23 | 	Args   []string
 24 | 	Stdin  string
 25 | 	Stdout string
 26 | }
 27 | 
 28 | var terminal *term.Term
 29 | var processGroups []uint32
 30 | 
 31 | var ForegroundPid uint32
 32 | var ForegroundProcess error = errors.New("Process is a foreground process")
 33 | var homedirRe *regexp.Regexp = regexp.MustCompile("^~([a-zA-Z]*)?(/*)?")
 34 | 
 35 | func main() {
 36 | 	// Initialize the terminal
 37 | 	t, err := term.Open("/dev/tty")
 38 | 	if err != nil {
 39 | 		panic(err)
 40 | 	}
 41 | 	// Restore the previous terminal settings at the end of the program
 42 | 	defer t.Restore()
 43 | 	t.SetCbreak()
 44 | 	terminal = t
 45 | 
 46 | 	child := make(chan os.Signal)
 47 | 	signal.Notify(child, syscall.SIGCHLD)
 48 | 	signal.Ignore(
 49 | 		syscall.SIGTTOU,
 50 | 		syscall.SIGINT,
 51 | 	)
 52 | 	os.Setenv("$", "$")
 53 | 	os.Setenv("SHELL", os.Args[0])
 54 | 	if u, err := user.Current(); err == nil {
 55 | 		SourceFile(u.HomeDir + "/.goshrc")
 56 | 	}
 57 | 	PrintPrompt()
 58 | 	r := bufio.NewReader(t)
 59 | 	var cmd Command
 60 | 	for {
 61 | 		c, _, err := r.ReadRune()
 62 | 		if err != nil {
 63 | 			fmt.Fprintf(os.Stderr, "%v\n", err)
 64 | 			continue
 65 | 		}
 66 | 		switch c {
 67 | 		case '\n':
 68 | 			// The terminal doesn't echo in raw mode,
 69 | 			// so print the newline itself to the terminal.
 70 | 			fmt.Printf("\n")
 71 | 
 72 | 			if cmd == "exit" || cmd == "quit" {
 73 | 				t.Restore()
 74 | 				os.Exit(0)
 75 | 			} else if cmd == "" {
 76 | 				PrintPrompt()
 77 | 			} else {
 78 | 				err := cmd.HandleCmd()
 79 | 				if err == ForegroundProcess {
 80 | 					Wait(child)
 81 | 				} else if err != nil {
 82 | 					fmt.Fprintf(os.Stderr, "%v\n", err)
 83 | 				}
 84 | 				PrintPrompt()
 85 | 			}
 86 | 			cmd = ""
 87 | 		case '\u0004':
 88 | 			if len(cmd) == 0 {
 89 | 				os.Exit(0)
 90 | 			}
 91 | 			err := cmd.Complete()
 92 | 			if err != nil {
 93 | 				fmt.Fprintf(os.Stderr, "%v\n", err)
 94 | 			}
 95 | 
 96 | 		case '\u007f', '\u0008':
 97 | 			if len(cmd) > 0 {
 98 | 				cmd = cmd[:len(cmd)-1]
 99 | 				fmt.Printf("\u0008 \u0008")
100 | 			}
101 | 		case '\t':
102 | 			err := cmd.Complete()
103 | 			if err != nil {
104 | 				fmt.Fprintf(os.Stderr, "%v\n", err)
105 | 			}
106 | 		default:
107 | 			fmt.Printf("%c", c)
108 | 			cmd += Command(c)
109 | 		}
110 | 	}
111 | }
112 | func (c Command) HandleCmd() error {
113 | 	parsed := c.Tokenize()
114 | 	if len(parsed) == 0 {
115 | 		// There was no command, it's not an error, the user just hit
116 | 		// enter.
117 | 		PrintPrompt()
118 | 		return nil
119 | 	}
120 | 	args := make([]string, 0, len(parsed))
121 | 	for _, val := range parsed[1:] {
122 | 		args = append(args, os.ExpandEnv(val))
123 | 	}
124 | 	// newargs will be at least len(parsed in size, so start by allocating a slice
125 | 	// of that capacity
126 | 	newargs := make([]string, 0, len(args))
127 | 	for _, token := range args {
128 | 		token = replaceTilde(token)
129 | 		expanded, err := filepath.Glob(token)
130 | 		if err != nil || len(expanded) == 0 {
131 | 			newargs = append(newargs, token)
132 | 			continue
133 | 		}
134 | 		newargs = append(newargs, expanded...)
135 | 
136 | 	}
137 | 	args = newargs
138 | 	var backgroundProcess bool
139 | 	if parsed[len(parsed)-1] == "&" {
140 | 		// Strip off the &, it's not part of the command.
141 | 		parsed = parsed[:len(parsed)-1]
142 | 		backgroundProcess = true
143 | 	}
144 | 	switch parsed[0] {
145 | 	case "cd":
146 | 		if len(args) == 0 {
147 | 			return fmt.Errorf("Must provide an argument to cd")
148 | 		}
149 | 		old, _ := os.Getwd()
150 | 		err := os.Chdir(args[0])
151 | 		if err == nil {
152 | 			new, _ := os.Getwd()
153 | 			os.Setenv("PWD", new)
154 | 			os.Setenv("OLDPWD", old)
155 | 		}
156 | 		return err
157 | 	case "set":
158 | 		if len(args) != 2 {
159 | 			return fmt.Errorf("Usage: set var value")
160 | 		}
161 | 		return os.Setenv(args[0], args[1])
162 | 	case "source":
163 | 		if len(args) < 1 {
164 | 			return fmt.Errorf("Usage: source file [...other files]")
165 | 		}
166 | 
167 | 		for _, f := range args {
168 | 			SourceFile(f)
169 | 		}
170 | 		return nil
171 | 	case "jobs":
172 | 		fmt.Printf("Job listing:\n\n")
173 | 		for i, leader := range processGroups {
174 | 			fmt.Printf("Job %d (%d)\n", i, leader)
175 | 		}
176 | 		return nil
177 | 	case "bg":
178 | 		if len(args) < 1 {
179 | 			return fmt.Errorf("Must specify job to background.")
180 | 		}
181 | 		i, err := strconv.Atoi(args[0])
182 | 		if err != nil {
183 | 			return err
184 | 		}
185 | 
186 | 		if i >= len(processGroups) || i < 0 {
187 | 			return fmt.Errorf("Invalid job id %d", i)
188 | 		}
189 | 		p, err := os.FindProcess(int(processGroups[i]))
190 | 		if err != nil {
191 | 			return err
192 | 		}
193 | 		if err := p.Signal(syscall.SIGCONT); err != nil {
194 | 			return err
195 | 		}
196 | 		return nil
197 | 	case "fg":
198 | 		if len(args) < 1 {
199 | 			return fmt.Errorf("Must specify job to foreground.")
200 | 		}
201 | 		i, err := strconv.Atoi(args[0])
202 | 		if err != nil {
203 | 			return err
204 | 		}
205 | 
206 | 		if i >= len(processGroups) || i < 0 {
207 | 			return fmt.Errorf("Invalid job id %d", i)
208 | 		}
209 | 		p, err := os.FindProcess(int(processGroups[i]))
210 | 		if err != nil {
211 | 			return err
212 | 		}
213 | 		if err := p.Signal(syscall.SIGCONT); err != nil {
214 | 			return err
215 | 		}
216 | 		terminal.Restore()
217 | 		var pid uint32 = processGroups[i]
218 | 		_, _, err3 := syscall.RawSyscall(
219 | 			syscall.SYS_IOCTL,
220 | 			uintptr(0),
221 | 			uintptr(syscall.TIOCSPGRP),
222 | 			uintptr(unsafe.Pointer(&pid)),
223 | 		)
224 | 		if err3 != syscall.Errno(0) {
225 | 			panic(fmt.Sprintf("Err: %v", err3))
226 | 		} else {
227 | 			ForegroundPid = pid
228 | 			return ForegroundProcess
229 | 		}
230 | 
231 | 	case "autocomplete":
232 | 		if len(args) < 2 {
233 | 			return fmt.Errorf("Usage: autocomplete regex value [more values...]")
234 | 		}
235 | 		if autocompletions == nil {
236 | 			autocompletions = make(map[*regexp.Regexp][]Token)
237 | 		}
238 | 		re, err := regexp.Compile(args[0])
239 | 		if err != nil {
240 | 			return err
241 | 		}
242 | 
243 | 		for _, t := range args[1:] {
244 | 			autocompletions[re] = append(autocompletions[re], Token(t))
245 | 		}
246 | 
247 | 		return nil
248 | 	}
249 | 	// Convert parsed from []string to []Token. We should refactor all the code
250 | 	// to use tokens, but for now just do this instead of going back and changing
251 | 	// all the references/declarations in every other section of code.
252 | 	var parsedtokens []Token = []Token{Token(parsed[0])}
253 | 	for _, t := range args {
254 | 		parsedtokens = append(parsedtokens, Token(t))
255 | 	}
256 | 	commands := ParseCommands(parsedtokens)
257 | 	var cmds []*exec.Cmd
258 | 	for i, c := range commands {
259 | 		if len(c.Args) == 0 {
260 | 			// This should have never happened, there is
261 | 			// no command, but let's avoid panicing.
262 | 			continue
263 | 		}
264 | 		newCmd := exec.Command(c.Args[0], c.Args[1:]...)
265 | 		newCmd.Stderr = os.Stderr
266 | 		cmds = append(cmds, newCmd)
267 | 
268 | 		// If there was an Stdin specified, use it.
269 | 		if c.Stdin != "" {
270 | 			// Open the file to convert it to an io.Reader
271 | 			if f, err := os.Open(c.Stdin); err == nil {
272 | 				newCmd.Stdin = f
273 | 				defer f.Close()
274 | 			}
275 | 		} else {
276 | 			// There was no Stdin specified, so
277 | 			// connect it to the previous process in the
278 | 			// pipeline if there is one, the first process
279 | 			// still uses os.Stdin
280 | 			if i > 0 {
281 | 				pipe, err := cmds[i-1].StdoutPipe()
282 | 				if err != nil {
283 | 					continue
284 | 				}
285 | 				newCmd.Stdin = pipe
286 | 			} else {
287 | 				newCmd.Stdin = os.Stdin
288 | 			}
289 | 		}
290 | 		// If there was a Stdout specified, use it.
291 | 		if c.Stdout != "" {
292 | 			// Create the file to convert it to an io.Reader
293 | 			if f, err := os.Create(c.Stdout); err == nil {
294 | 				newCmd.Stdout = f
295 | 				defer f.Close()
296 | 			}
297 | 		} else {
298 | 			// There was no Stdout specified, so
299 | 			// connect it to the previous process in the
300 | 			// unless it's the last command in the pipeline,
301 | 			// which still uses os.Stdout
302 | 			if i == len(commands)-1 {
303 | 				newCmd.Stdout = os.Stdout
304 | 			}
305 | 		}
306 | 	}
307 | 
308 | 	var pgrp uint32
309 | 	sysProcAttr := &syscall.SysProcAttr{
310 | 		Setpgid: true,
311 | 	}
312 | 	for _, c := range cmds {
313 | 		c.SysProcAttr = sysProcAttr
314 | 		if err := c.Start(); err != nil {
315 | 			return err
316 | 		}
317 | 		if sysProcAttr.Pgid == 0 {
318 | 			sysProcAttr.Pgid, _ = syscall.Getpgid(c.Process.Pid)
319 | 			pgrp = uint32(sysProcAttr.Pgid)
320 | 			processGroups = append(processGroups, uint32(c.Process.Pid))
321 | 		}
322 | 	}
323 | 	if backgroundProcess {
324 | 		// We can't tell if a background process returns an error
325 | 		// or not, so we just claim it didn't.
326 | 		return nil
327 | 	}
328 | 	ForegroundPid = pgrp
329 | 	terminal.Restore()
330 | 	_, _, err1 := syscall.RawSyscall(
331 | 		syscall.SYS_IOCTL,
332 | 		uintptr(0),
333 | 		uintptr(syscall.TIOCSPGRP),
334 | 		uintptr(unsafe.Pointer(&pgrp)),
335 | 	)
336 | 	// RawSyscall returns an int for the error, we need to compare
337 | 	// to syscall.Errno(0) instead of nil
338 | 	if err1 != syscall.Errno(0) {
339 | 		return err1
340 | 	}
341 | 	return ForegroundProcess
342 | }
343 | func PrintPrompt() {
344 | 	if p := os.Getenv("PROMPT"); p != "" {
345 | 		if len(p) > 1 && p[0] == '!' {
346 | 			input := os.ExpandEnv(p[1:])
347 | 			split := strings.Fields(input)
348 | 			cmd := exec.Command(split[0], split[1:]...)
349 | 			cmd.Stdout = os.Stderr
350 | 			if err := cmd.Run(); err != nil {
351 | 				if _, ok := err.(*exec.ExitError); !ok {
352 | 					// Fall back on our standard prompt, with a warning.
353 | 					fmt.Fprintf(os.Stderr, "\nInvalid prompt command\n> ")
354 | 				}
355 | 			}
356 | 		} else {
357 | 			fmt.Fprintf(os.Stderr, "\n%s", os.ExpandEnv(p))
358 | 		}
359 | 	} else {
360 | 		fmt.Fprintf(os.Stderr, "\n> ")
361 | 	}
362 | }
363 | func ParseCommands(tokens []Token) []ParsedCommand {
364 | 	// Keep track of the current command being built
365 | 	var currentCmd ParsedCommand
366 | 	// Keep array of all commands that have been built, so we can create the
367 | 	// pipeline
368 | 	var allCommands []ParsedCommand
369 | 	// Keep track of where this command started in parsed, so that we can build
370 | 	// currentCommand.Args when we find a special token.
371 | 	var lastCommandStart = 0
372 | 	// Keep track of if we've found a special token such as < or >, so that
373 | 	// we know if currentCmd.Args has already been populated.
374 | 	var foundSpecial bool
375 | 	var nextStdin, nextStdout bool
376 | 	for i, t := range tokens {
377 | 		if nextStdin {
378 | 			currentCmd.Stdin = string(t)
379 | 			nextStdin = false
380 | 		}
381 | 		if nextStdout {
382 | 			currentCmd.Stdout = string(t)
383 | 			nextStdout = false
384 | 		}
385 | 		if t.IsSpecial() || i == len(tokens)-1 {
386 | 			if foundSpecial == false {
387 | 				// Convert from Token to string
388 | 				var slice []Token
389 | 				if i == len(tokens)-1 {
390 | 					slice = tokens[lastCommandStart:]
391 | 				} else {
392 | 					slice = tokens[lastCommandStart:i]
393 | 				}
394 | 
395 | 				for _, t := range slice {
396 | 					currentCmd.Args = append(currentCmd.Args, string(t))
397 | 				}
398 | 			}
399 | 			foundSpecial = true
400 | 		}
401 | 		if t.IsStdinRedirect() {
402 | 			nextStdin = true
403 | 		}
404 | 		if t.IsStdoutRedirect() {
405 | 			nextStdout = true
406 | 		}
407 | 		if t.IsPipe() || i == len(tokens)-1 {
408 | 			allCommands = append(allCommands, currentCmd)
409 | 			lastCommandStart = i + 1
410 | 			foundSpecial = false
411 | 			currentCmd = ParsedCommand{}
412 | 		}
413 | 	}
414 | 	return allCommands
415 | }
416 | func SourceFile(filename string) error {
417 | 	f, err := os.Open(filename)
418 | 	if err != nil {
419 | 		return err
420 | 	}
421 | 	defer f.Close()
422 | 	scanner := bufio.NewReader(f)
423 | 	for {
424 | 		line, err := scanner.ReadString('\n')
425 | 		switch err {
426 | 		case io.EOF:
427 | 			return nil
428 | 		case nil:
429 | 			// Nothing special
430 | 		default:
431 | 			return err
432 | 		}
433 | 		c := Command(line)
434 | 		if err := c.HandleCmd(); err != nil {
435 | 			return err
436 | 		}
437 | 	}
438 | }
439 | func Wait(ch chan os.Signal) {
440 | 	for {
441 | 		select {
442 | 		case <-ch:
443 | 			newPg := make([]uint32, 0, len(processGroups))
444 | 			for _, pg := range processGroups {
445 | 				var status syscall.WaitStatus
446 | 				pid1, err := syscall.Wait4(int(pg), &status, syscall.WNOHANG|syscall.WUNTRACED|syscall.WCONTINUED, nil)
447 | 				if pid1 == 0 && err == nil {
448 | 					// We don't want to accidentally remove things from processGroups if there was an error
449 | 					// from wait.
450 | 					newPg = append(newPg, pg)
451 | 					continue
452 | 				}
453 | 				switch {
454 | 				case status.Continued():
455 | 					newPg = append(newPg, pg)
456 | 
457 | 					if ForegroundPid == 0 {
458 | 						terminal.Restore()
459 | 						var pid uint32 = pg
460 | 						_, _, err3 := syscall.RawSyscall(
461 | 							syscall.SYS_IOCTL,
462 | 							uintptr(0),
463 | 							uintptr(syscall.TIOCSPGRP),
464 | 							uintptr(unsafe.Pointer(&pid)),
465 | 						)
466 | 						if err3 != syscall.Errno(0) {
467 | 							panic(fmt.Sprintf("Err: %v", err3))
468 | 						}
469 | 						ForegroundPid = pid
470 | 					}
471 | 				case status.Stopped():
472 | 					newPg = append(newPg, pg)
473 | 					if pg == ForegroundPid && ForegroundPid != 0 {
474 | 						terminal.SetCbreak()
475 | 						var mypid uint32 = uint32(syscall.Getpid())
476 | 						_, _, err3 := syscall.RawSyscall(
477 | 							syscall.SYS_IOCTL,
478 | 							uintptr(0),
479 | 							uintptr(syscall.TIOCSPGRP),
480 | 							uintptr(unsafe.Pointer(&mypid)),
481 | 						)
482 | 						if err3 != syscall.Errno(0) {
483 | 							panic(fmt.Sprintf("Err: %v", err3))
484 | 						}
485 | 						ForegroundPid = 0
486 | 					}
487 | 					fmt.Fprintf(os.Stderr, "%v is stopped\n", pid1)
488 | 				case status.Signaled():
489 | 					if pg == ForegroundPid && ForegroundPid != 0 {
490 | 						terminal.SetCbreak()
491 | 						var mypid uint32 = uint32(syscall.Getpid())
492 | 						_, _, err3 := syscall.RawSyscall(
493 | 							syscall.SYS_IOCTL,
494 | 							uintptr(0),
495 | 							uintptr(syscall.TIOCSPGRP),
496 | 							uintptr(unsafe.Pointer(&mypid)),
497 | 						)
498 | 						if err3 != syscall.Errno(0) {
499 | 							panic(fmt.Sprintf("Err: %v", err3))
500 | 						}
501 | 						ForegroundPid = 0
502 | 					}
503 | 
504 | 					fmt.Fprintf(os.Stderr, "%v terminated by signal %v\n", pg, status.StopSignal())
505 | 				case status.Exited():
506 | 					if pg == ForegroundPid && ForegroundPid != 0 {
507 | 						terminal.SetCbreak()
508 | 						var mypid uint32 = uint32(syscall.Getpid())
509 | 						_, _, err3 := syscall.RawSyscall(
510 | 							syscall.SYS_IOCTL,
511 | 							uintptr(0),
512 | 							uintptr(syscall.TIOCSPGRP),
513 | 							uintptr(unsafe.Pointer(&mypid)),
514 | 						)
515 | 						if err3 != syscall.Errno(0) {
516 | 							panic(fmt.Sprintf("Err: %v", err3))
517 | 						}
518 | 						ForegroundPid = 0
519 | 					} else {
520 | 						fmt.Fprintf(os.Stderr, "%v exited (exit status: %v)\n", pid1, status.ExitStatus())
521 | 					}
522 | 					os.Setenv("?", strconv.Itoa(status.ExitStatus()))
523 | 				default:
524 | 					newPg = append(newPg, pg)
525 | 					fmt.Fprintf(os.Stderr, "Still running: %v: %v\n", pid1, status)
526 | 				}
527 | 			}
528 | 			processGroups = newPg
529 | 		}
530 | 
531 | 		if ForegroundPid == 0 {
532 | 			return
533 | 		}
534 | 	}
535 | }
536 | func replaceTilde(s string) string {
537 | 	if match := homedirRe.FindStringSubmatch(s); match != nil {
538 | 		var u *user.User
539 | 		var err error
540 | 		if match[1] != "" {
541 | 			u, err = user.Lookup(match[1])
542 | 		} else {
543 | 			u, err = user.Current()
544 | 		}
545 | 		if err == nil {
546 | 			return strings.Replace(s, match[0], u.HomeDir, 1)
547 | 		}
548 | 	}
549 | 	return s
550 | }
551 | 


--------------------------------------------------------------------------------
/prefix.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import ()
 4 | 
 5 | func LongestPrefix(strs []string) string {
 6 | 	if len(strs) == 0 {
 7 | 		return ""
 8 | 	}
 9 | 
10 | 	prefix := strs[0]
11 | 	for _, cmp := range strs[1:] {
12 | 		for i := range prefix {
13 | 			if i > len(cmp) || prefix[i] != cmp[i] {
14 | 				prefix = cmp[:i]
15 | 				break
16 | 			}
17 | 		}
18 | 	}
19 | 	return prefix
20 | }
21 | 


--------------------------------------------------------------------------------
/prefix_test.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"testing"
 5 | )
 6 | 
 7 | func TestLongestPrefix(t *testing.T) {
 8 | 	cases := []struct {
 9 | 		Val      []string
10 | 		Expected string
11 | 	}{
12 | 		// Empty case or nil slice
13 | 		{nil, ""},
14 | 		{[]string{}, ""},
15 | 
16 | 		// Prefix of 1 element is itself
17 | 		{[]string{"a"}, "a"},
18 | 
19 | 		// 2 elements with no prefix
20 | 		{[]string{"a", "b"}, ""},
21 | 
22 | 		// 2 elements with a common prefix
23 | 		{[]string{"aa", "ab"}, "a"},
24 | 
25 | 		// multiple elements
26 | 		{[]string{"aaaa", "aabb", "aaac"}, "aa"},
27 | 	}
28 | 	for i, tc := range cases {
29 | 		if got := LongestPrefix(tc.Val); got != tc.Expected {
30 | 			t.Errorf("Unexpected prefix for case %d: got %v want %v", i, got, tc.Expected)
31 | 		}
32 | 	}
33 | }
34 | 


--------------------------------------------------------------------------------
/tokenize.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"strings"
 5 | 	"unicode"
 6 | )
 7 | 
 8 | func (c Command) Tokenize() []string {
 9 | 	var parsed []string
10 | 	tokenStart := -1
11 | 	inStringLiteral := false
12 | 	for i, chr := range c {
13 | 		switch chr {
14 | 		case '\'':
15 | 			if inStringLiteral {
16 | 				if i > 0 && c[i-1] == '\\' {
17 | 					// The quote was escaped, so ignore it.
18 | 					continue
19 | 				}
20 | 				inStringLiteral = false
21 | 
22 | 				token := string(c[tokenStart:i])
23 | 
24 | 				// Replace escaped quotes with just a single ' before appending
25 | 				token = strings.Replace(token, `\'`, "'", -1)
26 | 				parsed = append(parsed, token)
27 | 
28 | 				// Now that we've finished, reset the tokenStart for the next token.
29 | 				tokenStart = -1
30 | 			} else {
31 | 				// This is the quote, which means the literal starts at the next
32 | 				// character
33 | 				tokenStart = i + 1
34 | 				inStringLiteral = true
35 | 			}
36 | 		case '|', '<', '>', '&':
37 | 			if inStringLiteral {
38 | 				continue
39 | 			}
40 | 			if tokenStart >= 0 {
41 | 				parsed = append(parsed, string(c[tokenStart:i]))
42 | 			}
43 | 			parsed = append(parsed, string(chr))
44 | 			tokenStart = -1
45 | 		default:
46 | 			if inStringLiteral {
47 | 				continue
48 | 			}
49 | 			if unicode.IsSpace(chr) {
50 | 				if tokenStart == -1 {
51 | 					continue
52 | 				}
53 | 				parsed = append(parsed, string(c[tokenStart:i]))
54 | 				tokenStart = -1
55 | 			} else if tokenStart == -1 {
56 | 				tokenStart = i
57 | 			}
58 | 		}
59 | 	}
60 | 	if tokenStart >= 0 {
61 | 		if inStringLiteral {
62 | 			// Ignore the ' character
63 | 			tokenStart += 1
64 | 		}
65 | 		parsed = append(parsed, string(c[tokenStart:]))
66 | 	}
67 | 	return parsed
68 | }
69 | 
70 | type Token string
71 | 
72 | func (t Token) IsPipe() bool {
73 | 	return t == "|"
74 | }
75 | 
76 | func (t Token) IsSpecial() bool {
77 | 	return t == "<" || t == ">" || t == "|"
78 | }
79 | 
80 | func (t Token) IsStdinRedirect() bool {
81 | 	return t == "<"
82 | }
83 | 
84 | func (t Token) IsStdoutRedirect() bool {
85 | 	return t == ">"
86 | }
87 | 


--------------------------------------------------------------------------------
/tokenize_test.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"testing"
 5 | )
 6 | 
 7 | func TestTokenization(t *testing.T) {
 8 | 	tests := []struct {
 9 | 		cmd      Command
10 | 		expected []string
11 | 	}{
12 | 		{cmd: "ls", expected: []string{"ls"}},
13 | 		{"     ls    	", []string{"ls"}},
14 | 		{"ls -l", []string{"ls", "-l"}},
15 | 		{"git commit -m 'I am message'", []string{"git", "commit", "-m", "I am message"}},
16 | 		{"git commit -m 'I\\'m another message'", []string{"git", "commit", "-m", "I'm another message"}},
17 | 		{"ls|cat", []string{"ls", "|", "cat"}},
18 | 	}
19 | 	for i, tc := range tests {
20 | 		val := tc.cmd.Tokenize()
21 | 		if len(val) != len(tc.expected) {
22 | 			// The below loop might panic if the lengths aren't equal, so this is fatal instead of an error.
23 | 			t.Fatalf("Mismatch for result length in test case %d. Got '%v' want '%v'", i, len(val), len(tc.expected))
24 | 		}
25 | 		for j, token := range val {
26 | 			if token != tc.expected[j] {
27 | 				t.Errorf("Mismatch for index %d in test case %d. Got '%v' want '%v'", j, i, token, tc.expected[j])
28 | 			}
29 | 		}
30 | 	}
31 | }
32 | func TestParseCommands(t *testing.T) {
33 | 	tests := []struct {
34 | 		val      []Token
35 | 		expected []ParsedCommand
36 | 	}{
37 | 		{
38 | 			[]Token{"ls"},
39 | 			[]ParsedCommand{
40 | 				ParsedCommand{[]string{"ls"}, "", ""},
41 | 			},
42 | 		},
43 | 		{
44 | 			[]Token{"ls", "|", "cat"},
45 | 			[]ParsedCommand{
46 | 				ParsedCommand{[]string{"ls"}, "", ""},
47 | 				ParsedCommand{[]string{"cat"}, "", ""},
48 | 			},
49 | 		},
50 | 		{
51 | 			[]Token{"ls", ">", "cat"},
52 | 			[]ParsedCommand{
53 | 				ParsedCommand{[]string{"ls"}, "", "cat"},
54 | 			},
55 | 		},
56 | 		{
57 | 			[]Token{"ls", "<", "cat"},
58 | 			[]ParsedCommand{
59 | 				ParsedCommand{[]string{"ls"}, "cat", ""},
60 | 			},
61 | 		},
62 | 		{
63 | 			[]Token{"ls", ">", "foo", "<", "bar", "|", "cat", "hello", ">", "x", "|", "tee"},
64 | 			[]ParsedCommand{
65 | 				ParsedCommand{[]string{"ls"}, "bar", "foo"},
66 | 				ParsedCommand{[]string{"cat", "hello"}, "", "x"},
67 | 				ParsedCommand{[]string{"tee"}, "", ""},
68 | 			},
69 | 		},
70 | 	}
71 | 
72 | 	for i, tc := range tests {
73 | 		val := ParseCommands(tc.val)
74 | 		if len(val) != len(tc.expected) {
75 | 			t.Fatalf("Unexpected number of ParsedCommands in test %d. Got %v want %v", i, val, tc.expected)
76 | 		}
77 | 		for j, _ := range val {
78 | 			if val[j].Stdin != tc.expected[j].Stdin {
79 | 				t.Fatalf("Mismatch for test %d Stdin. Got %v want %v", i, val[j].Stdin, tc.expected[j].Stdin)
80 | 			}
81 | 			if val[j].Stdout != tc.expected[j].Stdout {
82 | 				t.Fatalf("Mismatch for test %d Stdout. Got %v want %v", i, val[j].Stdout, tc.expected[j].Stdout)
83 | 			}
84 | 			for k, _ := range val[j].Args {
85 | 				if val[j].Args[k] != tc.expected[j].Args[k] {
86 | 					t.Fatalf("Mismatch for test %d. Got %v want %v", i, val[j].Args[k], tc.expected[j].Args[k])
87 | 				}
88 | 			}
89 | 		}
90 | 	}
91 | }
92 | 


--------------------------------------------------------------------------------