Fix various issues

autoload/do.vim

@@ -15,20 +15,20 @@ let s:do_refresh_key = "<C-L>"
 let s:do_update_time = 500
 let s:do_auto_show_process_window = 1
 
+""
 " Load Python script
-if filereadable($VIMRUNTIME."/plugin/python/do.py")
-  pyfile $VIMRUNTIME/plugin/do.py
-elseif filereadable($HOME."/.vim/plugin/python/do.py")
-  pyfile $HOME/.vim/plugin/python/do.py
+"
+" Search relative to this file.
+let $CUR_DIRECTORY=expand("<sfile>:p:h")
+
+if filereadable($CUR_DIRECTORY."/python/do.py")
+    if has("python")
+        pyfile $CUR_DIRECTORY/python/do.py
+    elseif has("python3")
+        py3file $CUR_DIRECTORY/python/do.py
+    endif
 else
-  " when we use pathogen for instance
-  let $CUR_DIRECTORY=expand("<sfile>:p:h")
-
-  if filereadable($CUR_DIRECTORY."/python/do.py")
-    pyfile $CUR_DIRECTORY/python/do.py
-  else
-    call confirm('vdebug.vim: Unable to find do.py. Place it in either your home vim directory or in the Vim runtime directory.', 'OK')
-  endif
+    call confirm('do.vim: Unable to find autoload/python/do.py. Place it in either your home vim directory or in the Vim runtime directory.', 'OK')
 endif
 
 ""
@@ -90,7 +90,11 @@ endfunction
 " function will reload them.
 "
 function! do#ReloadOptions()
-    python do_async.reload_options()
+    if has("python")
+        python do_async.reload_options()
+    elseif has("python3")
+        python3 do_async.reload_options()
+    endif
 endfunction
 
 ""
@@ -121,9 +125,11 @@ function! do#Execute(command, ...)
         call do#error("Supplied command is empty")
     else
         let s:previous_command = l:command
-python <<_EOF_
-do_async.execute(vim.eval("l:command"), int(vim.eval("l:quiet")) == 1)
-_EOF_
+        if has("python")
+            python do_async.execute(vim.eval("l:command"), int(vim.eval("l:quiet")) == 1)
+        elseif has("python3")
+            python3 do_async.execute(vim.eval("l:command"), int(vim.eval("l:quiet")) == 1)
+        endif
     endif
 endfunction
 
@@ -138,15 +144,145 @@ function! do#ExecuteSelection()
     call do#Execute(l:command)
 endfunction
 
+""
+" Keeps records on external processes started via do#ExecuteExternal .
+"
+let s:external_processes = {
+            \ 'by_id'  : {},
+            \ 'by_pid' : {},
+            \ }
+
+""
+" Default callback for external processes
+"
+" This is the default callback for external processes. It always excepts all
+" parameters and does nothing.
+"
+" @param string a:1 The ID of the external process
+" @param number or string a:2 The exit code of the external process
+"
+function! s:EmptyCallback(...)
+endfunction
+
+""
+" Start an external process
+"
+" Start an external process with the following options, which are given in a
+" Dict with fields:
+" - id (string): a user-defined ID
+" - split_output (integer): whether the output is to be split into stdout and
+"   stderr, the default is not to split the output
+" - callback (Funcref): a function to be called after the process is finished:
+"     function s:Callback(pid,exit_code)
+"       " ...
+"     endfunction
+"     options.callback = Function("s:Callback")
+"
+" @param string command The command to run
+" @param dict options The options as a dictionary
+"
+function! do#ExecuteExternal(command, options)
+    let record = {
+                \ 'id'           : get ( a:options, 'id', '' ),
+                \ 'command'      : a:command,
+                \ 'callback'     : get ( a:options, 'callback', function('s:EmptyCallback') ),
+                \ 'split_output' : get ( a:options, 'split_output', 0 ),
+                \ 'status'       : 'new',
+                \ 'pid'          : -1,
+                \ 'exit_code'    : -1,
+                \ }
+    if record.id != ''
+        let s:external_processes.by_id[record.id] = record
+    endif
+
+    let l:command = a:command
+    let l:pid     = -1
+    let l:split   = record.split_output
+    if empty(l:command)
+        "TODO: log
+        return 0
+    endif
+    let l:command = Strip(l:command)
+    if empty(l:command)
+        "TODO: log
+        return 0
+    else
+        let l:pid = pyeval ( 'do_async.execute(vim.eval("l:command"), external = True, split_output = vim.eval("l:split") == 1 )' )
+        let record.status = 'running'
+        let record.pid    = l:pid
+        let s:external_processes.by_pid[record.pid] = record
+    endif
+endfunction
+
+""
+" Get a previously started external process
+"
+" Returns the record of the last external process with the given ID. The
+" record is a Dict with fields:
+" - id (string): the user-defined ID
+" - command (string): the command
+" - split_output (integer): whether the output is split
+" - status (string): "new", "running", "finished", or "failed"
+" - pid (number): only valid while the status is "running"
+" - exit_code (number): only valid if the status is "finished"
+"
+" If a record with this ID does not exist, a record with the field 'status'
+" set to "failed" is returned.
+"
+" @param string id The ID of the process
+"
+function! do#GetExternal(id)
+    if ! has_key ( s:external_processes.by_id, a:id )
+        return { 'status' : 'failed', }
+    endif
+    return s:external_processes.by_id[a:id]
+endfunction
+
+""
+" Internal use: An external process is finished
+"
+" This function is called by Python after an external process finished. It
+" should not be called by a user.
+"
+" @param number pid The PID of the process, for identification
+" @param number exit_code The exit code
+"
+function! do#HookProcessFinished(pid,exit_code)
+    if has_key ( s:external_processes.by_pid, a:pid )
+        let record = s:external_processes.by_pid[a:pid]
+        let record.status    = 'finished'
+        let record.exit_code = a:exit_code
+        call remove ( s:external_processes.by_pid, a:pid )
+
+        call call ( record.callback, [ record.id, record.exit_code ] )
+
+        if record.split_output
+            " :TODO:14.07.2016 19:03:WM: only get the output when requested,
+            " this runs during an autocmd and takes to much time
+            let l = pyeval ( 'do_async.get_by_pid('.record.pid.').output().all_std()' )
+            let record.output_std = join ( l, "" )
+            let l = pyeval ( 'do_async.get_by_pid('.record.pid.').output().all_err()' )
+            let record.output_err = join ( l, "" )
+        else
+            " :TODO:14.07.2016 19:03:WM: only get the output when requested,
+            " this runs during an autocmd and takes to much time
+            let l = pyeval ( 'do_async.get_by_pid('.record.pid.').output().all()' )
+            let record.output = join ( l, "" )
+        endif
+    endif
+endfunction
+
 ""
 " Enable the file logger for debugging purposes.
 "
 " @param string file_path The path to the file to write log information
 "
 function! do#EnableLogger(file_path)
-python <<_EOF_
-do_async.enable_logger(vim.eval("a:file_path"))
-_EOF_
+    if has("python")
+        python do_async.enable_logger(vim.eval("a:file_path"))
+    elseif has("python3")
+        python3 do_async.enable_logger(vim.eval("a:file_path"))
+    endif
 endfunction
 
 ""
@@ -155,7 +291,11 @@ endfunction
 " The command window details currently running and finished processes.
 "
 function! do#ToggleCommandWindow()
-    python do_async.toggle_command_window()
+    if has("python")
+        python do_async.toggle_command_window()
+    elseif has("python3")
+        python3 do_async.toggle_command_window()
+    endif
 endfunction
 
 ""
@@ -164,7 +304,11 @@ endfunction
 " Executed automatically via an autocommand.
 "
 function! do#MarkCommandWindowAsClosed()
-    python do_async.mark_command_window_as_closed()
+    if has("python")
+        python do_async.mark_command_window_as_closed()
+    elseif has("python3")
+        python3 do_async.mark_command_window_as_closed()
+    endif
 endfunction
 
 ""
@@ -173,14 +317,22 @@ endfunction
 " Executed automatically via an autocommand.
 "
 function! do#MarkProcessWindowAsClosed()
-    python do_async.mark_process_window_as_closed()
+    if has("python")
+        python do_async.mark_process_window_as_closed()
+    elseif has("python3")
+        python3 do_async.mark_process_window_as_closed()
+    endif
 endfunction
 
 ""
 " Trigger selection of a process in the command window.
 "
 function! do#ShowProcessFromCommandWindow()
-    python do_async.show_process_from_command_window()
+    if has("python")
+        python do_async.show_process_from_command_window()
+    elseif has("python3")
+        python3 do_async.show_process_from_command_window()
+    endif
 endfunction
 
 ""
@@ -204,12 +356,21 @@ function! do#AssignAutocommands()
     execute "nnoremap <silent> " . do#get("do_refresh_key") . " :call do#nop()<CR>"
     execute "inoremap <silent> " . do#get("do_refresh_key") . ' <C-O>:call do#nop()<CR>'
     augroup vim_do
-        au CursorHold * python do_async.check()
-        au CursorHoldI * python do_async.check()
-        au CursorMoved * python do_async.check()
-        au CursorMovedI * python do_async.check()
-        au FocusGained * python do_async.check()
-        au FocusLost * python do_async.check()
+        if has("python")
+            au CursorHold   * python do_async.check()
+            au CursorHoldI  * python do_async.check()
+            au CursorMoved  * python do_async.check()
+            au CursorMovedI * python do_async.check()
+            au FocusGained  * python do_async.check()
+            au FocusLost    * python do_async.check()
+        elseif has("python3")
+            au CursorHold   * python3 do_async.check()
+            au CursorHoldI  * python3 do_async.check()
+            au CursorMoved  * python3 do_async.check()
+            au CursorMovedI * python3 do_async.check()
+            au FocusGained  * python3 do_async.check()
+            au FocusLost    * python3 do_async.check()
+        endif
     augroup END
     let &updatetime=do#get("do_update_time")
 endfunction
@@ -247,5 +408,12 @@ function! s:getVisualSelection()
 endfunction
 
 " Initialize do
-python do_async = Do()
-autocmd VimLeavePre * python do_async.stop()
+if has("python")
+    python do_async = Do()
+    autocmd VimLeavePre * python do_async.stop()
+elseif has("python3")
+    python3 do_async = Do()
+    autocmd VimLeavePre * python3 do_async.stop()
+endif
+
+" vim: expandtab: tabstop=4: shiftwidth=4

autoload/python/async.py

@@ -1,10 +1,13 @@
-import Queue
 import threading
 import subprocess
-import shlex
 import select
+import sys
 from utils import log
-import os
+
+if sys.version[0] == '2':
+    import Queue as queue
+else:
+    import queue as queue
 
 class AsyncProcessReader(threading.Thread):
     def __init__(self, process, output_q):
@@ -28,22 +31,36 @@ def _readfds(self):
         fds = [self.__process.stdout.fileno(), self.__process.stderr.fileno()]
         streams = [self.__process.stdout, self.__process.stderr]
 
+        # while the process is still alive ...
         while self.__process.poll() is None:
+            # wait for one of the file descriptors to be ready for reading
             fdsin, _, _ = select.select(fds, [], [])
             for fd in fdsin:
+                # read a line from the file descriptor
                 output = [None, None]
                 ind = fds.index(fd)
                 stream = streams[ind]
                 s = stream.readline()
                 if len(s) > 0:
                     output[ind] = s
                     yield output
+        # after the process has finished ...
+        for ind, stream in enumerate( streams ):
+            # read the rest of the output from the file descriptor
+            output = [None, None]
+            while True:
+                s = stream.readline()
+                if len(s) > 0:
+                    output[ind] = s
+                    yield output
+                else:
+                    break
 
 
 class ProcessPool:
     def __init__(self):
         self.__threads = []
-        self.__output_q = Queue.Queue(0)
+        self.__output_q = queue.Queue(0)
 
     def execute(self, cmd):
         subproc = subprocess.Popen(cmd, shell=True,
@@ -66,7 +83,7 @@ def get_outputs(self):
         try:
             for result in iter(self.__output_q.get_nowait, None):
                 results.append(result)
-        except Queue.Empty:
+        except queue.Empty:
             pass
 
         return results
@@ -79,3 +96,4 @@ def stop(self):
             for t in self.__threads:
                 t.join(1000)
 
+# vim: expandtab: tabstop=4: shiftwidth=4

autoload/python/buffer.py

@@ -18,8 +18,13 @@ def write(self, msg, overwrite):
         else:
             to_write = str(msg).split('\n')
 
-        if len(to_write) == 1 and to_write[0] == "":
-            return (last_line, last_line)
+        if len(to_write) == 0 or len(to_write) == 1 and to_write[0] == "":
+            # nothing to write, check overwrite
+            if overwrite:
+                self._buffer[:] = None
+                return (1, 1)
+            else:
+                return (last_line, last_line)
 
         if overwrite or self.is_empty():
             self._buffer[:] = to_write
@@ -131,4 +136,4 @@ def contents(self):
     def is_empty(self):
         return not self._buffer
 
-
+# vim: expandtab: tabstop=4: shiftwidth=4