WIP: buffer map approach

benchmarks/parse-file.el

@@ -5,19 +5,42 @@
 (let ((here (file-name-directory (or load-file-name buffer-file-name))))
 
   (with-temp-buffer
-    (setq-local phpinspect-current-buffer (phpinspect-make-buffer :buffer (current-buffer)))
+    ;; (setq-local phpinspect-current-buffer (phpinspect-make-buffer :buffer (current-buffer)))
 
     (insert-file-contents (concat here "/Response.php"))
 
-    (message "Metadata parse:")
-    (benchmark 1 '(phpinspect-buffer-parse phpinspect-current-buffer))
+    (message "Bmap warmup parse:")
+    (phpinspect-with-parse-context (phpinspect-make-pctx :incremental t)
+      (benchmark 1 '(phpinspect-parse-current-buffer)))
 
-    (message "Bare parse:")
+    (let ((bmap (phpinspect-make-bmap))
+          (bmap2 (phpinspect-make-bmap)))
+      (message "Bmap parse:")
+      (phpinspect-with-parse-context (phpinspect-make-pctx :incremental t :bmap bmap)
+        (benchmark 1 '(phpinspect-parse-current-buffer)))
+
+      (message "Bmap parse incremental:")
+      (phpinspect-with-parse-context (phpinspect-make-pctx :incremental t :bmap bmap2 :previous-bmap bmap :edtrack (phpinspect-make-edtrack))
+        (benchmark 1 '(phpinspect-parse-current-buffer)))
+
+      (message "Bmap parse incremental repeat:")
+      (phpinspect-with-parse-context (phpinspect-make-pctx :incremental t :previous-bmap bmap2 :edtrack (phpinspect-make-edtrack))
+        (benchmark 1 '(phpinspect-parse-current-buffer))))
+
+    ;; (message "Metadata parse:")
+    ;; (benchmark 1 '(phpinspect-buffer-parse phpinspect-current-buffer))
+
+    (message "Bare warmup parse:")
     (benchmark 1 '(phpinspect-parse-current-buffer))
 
-    (goto-char (floor (/ (point-max) 2 )))
-    (insert "abc")
-    (phpinspect-buffer-register-edit phpinspect-current-buffer (- (point) 3) (point) 0)
 
-    (message "Metadata parse incremental:")
-    (benchmark 1 '(phpinspect-buffer-parse phpinspect-current-buffer))))
+    (message "Bare parse:")
+    (benchmark 1 '(phpinspect-parse-current-buffer))))
+
+
+    ;; (goto-char (floor (/ (point-max) 2 )))
+    ;; (insert "abc")
+    ;; (phpinspect-buffer-register-edit phpinspect-current-buffer (- (point) 3) (point) 0)
+
+    ;; (message "Metadata parse incremental:")
+    ;; (benchmark 1 '(phpinspect-buffer-parse phpinspect-current-buffer))))

phpinspect-bmap.el

@@ -0,0 +1,205 @@
+;;; phpinspect-bmap.el --- PHP parsing and completion package  -*- lexical-binding: t; -*-
+
+;; Copyright (C) 2021  Free Software Foundation, Inc
+
+;; Author: Hugo Thunnissen <devel@hugot.nl>
+;; Keywords: php, languages, tools, convenience
+;; Version: 0
+
+;; This program is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; This program is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;;; Code:
+
+(cl-defstruct (phpinspect-bmap (:constructor phpinspect-make-bmap))
+  (starts (make-hash-table :test #'eql
+                           :size (floor (/ (point-max) 4))
+                           :rehash-size 1.5))
+  (ends (make-hash-table :test #'eql
+                           :size (floor (/ (point-max) 4))
+                           :rehash-size 1.5))
+  (meta (make-hash-table :test #'eq
+                           :size (floor (/ (point-max) 4))
+                           :rehash-size 1.5))
+  (overlays nil
+            :type list)
+  (current-row nil
+               :type list)
+  (current-row-start nil
+                     :type integer)
+  (current-row-end nil
+                   :type integer)
+  (whitespace-before ""
+                     :type string))
+
+(defsubst phpinspect--make-meta (parent start end whitespace-before token &optional overlayed)
+  (list parent start end whitespace-before token overlayed))
+
+(defsubst phpinspect--meta-parent (meta)
+  (car meta))
+
+(gv-define-setter phpinspect--meta-end (end meta) `(setcar (cddr ,meta) ,end))
+(gv-define-setter phpinspect--meta-start (start meta) `(setcar (cdr ,meta) ,start))
+(gv-define-setter phpinspect--meta-overlayed (overlayed meta) `(setcar (nthcdr 5 ,meta) ,overlayed))
+
+(defsubst phpinspect--meta-overlayed-p (meta overlay)
+  (eq (phpinspect--meta-overlayed meta) overlay))
+
+(defsubst phpinspect--meta-overlayed (meta)
+  (car (nthcdr 5 meta)))
+
+(defsubst phpinspect--meta-token (meta)
+  (car (cddddr meta)))
+
+(defsubst phpinspect--meta-end (meta)
+  (caddr meta))
+
+(defsubst phpinspect--meta-start (meta)
+  (cadr meta))
+
+(defsubst phpinspect--meta-overlaps-point (meta point)
+  (and (> (phpinspect--meta-end meta) point)
+       (<= (phpinspect--meta-start meta) point)))
+
+(defsubst phpinspect-bmap-register-whitespace (bmap whitespace)
+  (setf (phpinspect-bmap-whitespace-before bmap) whitespace))
+
+(defsubst phpinspect-bmap-register (bmap start end token)
+  (let* ((starts (phpinspect-bmap-starts bmap))
+         (ends (phpinspect-bmap-ends bmap))
+         (meta (phpinspect-bmap-meta bmap))
+         (current-row (phpinspect-bmap-current-row bmap))
+         (current-row-start (phpinspect-bmap-current-row-start bmap))
+         (current-row-end (phpinspect-bmap-current-row-end bmap))
+         (existing-end (gethash end ends))
+         (whitespace-before (phpinspect-bmap-whitespace-before bmap))
+         (token-meta (phpinspect--make-meta nil start end whitespace-before token)))
+    (setf (phpinspect-bmap-whitespace-before bmap) "")
+
+    (puthash start token-meta starts)
+
+    (if existing-end
+        (push token existing-end)
+      (puthash end (list token-meta) ends))
+
+    (puthash token token-meta meta)
+
+    (cond ((not current-row-start)
+           (setf (phpinspect-bmap-current-row-start bmap) start)
+           (setf (phpinspect-bmap-current-row-end bmap) end))
+          ((and (>= end current-row-end)
+                (<= start current-row-start))
+           (dolist (child current-row)
+             ;; Set parent
+             (setcar child token-meta))
+           (setf (phpinspect-bmap-current-row-start bmap) nil)
+           (setf (phpinspect-bmap-current-row bmap) nil))
+          ((> end current-row-end)
+           (setf (phpinspect-bmap-current-row-end bmap) end)))
+
+    (push token-meta (phpinspect-bmap-current-row bmap))))
+
+(defsubst phpinspect-overlay-p (overlay)
+  (eq 'overlay (car overlay)))
+
+(defsubst phpinspect-overlay-wrap-meta (overlay meta)
+  (when meta
+    (setq meta (cl-copy-list meta))
+    (setf (phpinspect--meta-start meta)
+          (+ (phpinspect--meta-start meta) (phpinspect-overlay-delta overlay)))
+    (setf (phpinspect--meta-end meta)
+          (+ (phpinspect--meta-end meta) (phpinspect-overlay-delta overlay)))
+    (setf (phpinspect--meta-overlayed meta) overlay)
+
+    meta))
+
+(cl-defmethod phpinspect-bmap-token-starting-at ((overlay (head overlay)) point)
+  (phpinspect-overlay-wrap-meta
+   overlay
+   (phpinspect-bmap-token-starting-at
+    (phpinspect-overlay-bmap overlay) (- point (phpinspect-overlay-delta overlay)))))
+
+(cl-defmethod phpinspect-bmap-token-starting-at ((bmap phpinspect-bmap) point)
+  (let ((overlay (phpinspect-bmap-overlay-at-point bmap point)))
+    (if overlay
+          (phpinspect-bmap-token-starting-at overlay point)
+      (gethash point (phpinspect-bmap-starts bmap)))))
+
+(defsubst phpinspect-bmap-overlay-at-point (bmap point)
+  (catch 'found
+    (dolist (overlay (phpinspect-bmap-overlays bmap))
+      (when (phpinspect-overlay-overlaps-point overlay point)
+        (throw 'found overlay)))))
+
+;; (cl-defmethod phpinspect-bmap-tokens-overlapping ((bmap phpinspect-bmap) point)
+;;   (
+
+(cl-defmethod phpinspect-bmap-token-meta ((overlay (head overlay)) token)
+  (phpinspect-bmap-token-meta (phpinspect-overlay-bmap overlay) token))
+
+(cl-defmethod phpinspect-bmap-token-meta ((bmap phpinspect-bmap) token)
+  (or (gethash token (phpinspect-bmap-meta bmap))
+      (let ((found?))
+        (catch 'found
+          (dolist (overlay (phpinspect-bmap-overlays bmap))
+            (when (setq found? (phpinspect-bmap-token-meta overlay token))
+              (throw 'found found?)))))))
+
+(defsubst phpinspect-probably-token-p (token)
+  (and (listp token)
+       (symbolp (car token))))
+
+(defsubst phpinspect-overlay-overlaps-point (overlay point)
+  (and (> (phpinspect-overlay-end overlay) point)
+       (<= (phpinspect-overlay-start overlay) point)))
+
+(defsubst phpinspect-overlay-bmap (overlay)
+  (car (nthcdr 4 overlay)))
+
+(defsubst phpinspect-overlay-delta (overlay)
+  (cadddr overlay))
+
+(defsubst phpinspect-overlay-start (overlay)
+  (cadr overlay))
+
+(defsubst phpinspect-overlay-end (overlay)
+  (caddr overlay))
+
+(defsubst phpinspect-bmap-overlay (bmap bmap-overlay token-meta pos-delta)
+  (let* ((overlays (phpinspect-bmap-overlays bmap))
+         (start (+ (phpinspect--meta-start token-meta) pos-delta))
+         (end (+ (phpinspect--meta-end token-meta) pos-delta))
+         (overlay `(overlay ,start ,end ,pos-delta ,bmap-overlay))
+         (before))
+    (phpinspect-bmap-register bmap start end (phpinspect--meta-token token-meta))
+
+    (if overlays
+        (progn
+          (catch 'break
+            (while (setq before (car overlays))
+              (if (> (phpinspect-overlay-start overlay) (phpinspect-overlay-end before))
+                  (throw 'break nil)
+                (setq overlays (cdr overlays)))))
+
+          (if before
+              ;; Append after
+              (setcdr overlays (cons overlay (cdr overlays)))
+            ;; Append at end of overlay list
+            (setcdr (last (phpinspect-bmap-overlays bmap)) overlay)))
+
+      (push overlay (phpinspect-bmap-overlays bmap)))))
+
+(provide 'phpinspect-bmap)
+;;; phpinspect-bmap.el ends here

phpinspect-edtrack.el

@@ -1,181 +1,162 @@
+;;; phpinspect-edtrack.el --- PHP parsing and completion package  -*- lexical-binding: t; -*-
 
+;; Copyright (C) 2021  Free Software Foundation, Inc
 
-(cl-defstruct (phpinspect-edtrack (:constructor phpinspect-make-edtrack))
-  (edits (phpinspect-make-ll)
-         :documentation "Sorted list of edits in buffer")
-  (taint-pool (phpinspect-make-tree :grow-root t)
-              :documentation "Non overlapping pool of tainted buffer regions"))
-
-
-(cl-defstruct (phpinspect-edit (:constructor phpinspect-make-edit))
-  (list nil
-        :type phpinspect-llnode)
-  (original-start 0
-                  :type integer)
-  (local-delta 0
-               :type integer)
-  (length 0))
-
-(defsubst phpinspect-edit-link (edit)
-  (when (phpinspect-edit-list edit)
-    (phpinspect-ll-link (phpinspect-edit-list edit) edit)))
-
-(defsubst phpinspect-edit--left-delta (edit)
-  (let* ((link (phpinspect-edit-link edit))
-         (left (when link (phpinspect-llnode-left (phpinspect-edit-link edit)))))
-    (if left (phpinspect-edit-delta (phpinspect-llnode-value left)) 0)))
-
-(cl-defmethod phpinspect-edit-start ((edit phpinspect-edit))
-  (+ (phpinspect-edit-original-start edit) (phpinspect-edit--left-delta edit)))
-
-(cl-defmethod phpinspect-edit-delta ((edit phpinspect-edit))
-  (+ (phpinspect-edit--left-delta edit) (phpinspect-edit-local-delta edit)))
-
-(cl-defmethod phpinspect-edit-end ((edit phpinspect-edit))
-  (let ((end (+ (phpinspect-edit-start edit)
-                (+ (phpinspect-edit-length edit) (phpinspect-edit-local-delta edit)))))
-    (if (> (phpinspect-edit-start edit) end)
-        (phpinspect-edit-start edit)
-      end)))
-
-(cl-defmethod phpinspect-edit-overlaps-point ((edit phpinspect-edit) (point integer))
-  (and (> (phpinspect-edit-end edit) point)
-       (<= (phpinspect-edit-start edit) point)))
-
-(cl-defmethod phpinspect-edit-before-point ((edit phpinspect-edit) (point integer))
-  (<= (phpinspect-edit-end edit) point))
-
-(cl-defmethod phpinspect-edit-before-original-point ((edit phpinspect-edit) (point integer))
-  (< (phpinspect-edit-original-end edit) point))
-
-(defsubst phpinspect-edit-overlaps (edit start end)
-  (let ((region (phpinspect-make-region start end)))
-    (or (phpinspect-edit-overlaps-point edit start)
-        (phpinspect-edit-overlaps-point edit (- end 1))
-        (phpinspect-region-overlaps-point region (phpinspect-edit-start edit))
-        (phpinspect-region-overlaps-point region (- (phpinspect-edit-end edit) 1)))))
-
-(defsubst phpinspect-edit-original-end (edit)
-  (+ (phpinspect-edit-original-start edit)
-     (+ (phpinspect-edit-length edit))))
-
-(cl-defmethod phpinspect-edit-overlaps-original-point (edit point)
-  (and (> (phpinspect-edit-original-end edit) point)
-       (<= (phpinspect-edit-original-start edit) point)))
+;; Author: Hugo Thunnissen <devel@hugot.nl>
+;; Keywords: php, languages, tools, convenience
+;; Version: 0
 
-(defsubst phpinspect-edit-overlaps-original (edit start end)
-  (let ((region (phpinspect-make-region start end)))
-    (or (phpinspect-edit-overlaps-original-point edit start)
-        (phpinspect-edit-overlaps-original-point edit (- end 1))
-        (phpinspect-region-overlaps-point region (phpinspect-edit-original-start edit))
-        (phpinspect-region-overlaps-point region (- (phpinspect-edit-original-end edit) 1)))))
+;; This program is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
 
-(defsubst phpinspect-edtrack-clear-taints (tracker)
-  (setf (phpinspect-edtrack-taint-pool tracker) (phpinspect-make-tree :grow-root t)))
+;; This program is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
 
-(defsubst phpinspect-edtrack-clear (tracker)
-  (phpinspect-edtrack-clear-taints tracker)
-  (setf (phpinspect-edtrack-edits tracker) (phpinspect-make-ll)))
+;; You should have received a copy of the GNU General Public License
+;; along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-(defsubst phpinspect-edtrack-has-taints-p (tracker)
-  (not (phpinspect-tree-empty-p (phpinspect-edtrack-taint-pool tracker))))
+;;; Commentary:
 
-(defsubst phpinspect-edtrack-register-taint (tracker start end)
-  (let* ((pool (phpinspect-edtrack-taint-pool tracker))
-         (overlappers (phpinspect-tree-find-overlapping-children pool start end)))
+;;; Code:
 
-    (when overlappers
-      (seq-doseq (overlapper overlappers)
-        (when (> (phpinspect-tree-end overlapper) end)
-          (setq end (phpinspect-tree-end overlapper)))
-
-        (when (< (phpinspect-tree-start overlapper) start)
-          (setq start (phpinspect-tree-start overlapper))))
-      (phpinspect-slice-detach overlappers))
-
-    (phpinspect-tree-insert pool start end 'edited)))
+(cl-defstruct (phpinspect-edtrack (:constructor phpinspect-make-edtrack))
+  (edits nil
+         :type list)
+  (taint-pool nil
+              :type list))
 
-(cl-defmethod phpinspect-edtrack-register-edit
-  ((tracker phpinspect-edtrack) (start integer) (end integer) (pre-change-length integer))
-  (let* ((edits (phpinspect-edtrack-edits tracker))
-         (first-overlap)
-         (last-overlap)
-         (edit-before)
-         (new-edit))
+(defsubst phpinspect-edtrack-make-taint-iterator (track)
+  (cons (car (phpinspect-edtrack-taint-pool track))
+        (cl-copy-list (cdr (phpinspect-edtrack-taint-pool track)))))
 
-    (catch 'break
-      (seq-doseq (edit edits)
-        (cond
-         ((phpinspect-edit-before-point edit start)
-          (setq edit-before edit))
-         (edit-before
-          (throw 'break nil)))))
-
-    (if edit-before
-        (setq new-edit (phpinspect-make-edit
-                        :original-start (- start (phpinspect-edit-delta edit-before))
-                        :local-delta (- (- end start) pre-change-length)
-                        :list edits
-                        :length pre-change-length))
-      (setq new-edit (phpinspect-make-edit
-                      :original-start start
-                      :local-delta (- (- end start) pre-change-length)
-                      :list edits
-                      :length pre-change-length)))
-
-    (if edit-before
-        (phpinspect-ll-insert-right (phpinspect-edit-link edit-before) new-edit)
-      (phpinspect-ll-push new-edit edits))
-
-    (phpinspect-edtrack-register-taint
-     tracker (phpinspect-edit-original-start new-edit) (phpinspect-edit-original-end new-edit))
-
-    ;; Return
-    new-edit))
-
-(defsubst phpinspect-edtrack--last-edit-before-point (edtrack point)
-  (let ((found))
-    (catch 'break
-        (seq-doseq (edit (phpinspect-edtrack-edits edtrack))
-          (if (phpinspect-edit-before-point edit point)
-              (setq found edit)
-            (when found
-              (throw 'break nil)))))
+(defsubst phpinspect-taint-iterator-token-is-tainted-p (iter meta)
+  (let ((current (car iter)))
+    (when current
+      (while (and current (> (phpinspect--meta-start meta) (phpinspect-taint-end current)))
+        (setq current (pop (cdr iter))))
 
-    found))
+      (and current (phpinspect-taint-overlaps-meta current meta)))))
 
-(defsubst phpinspect-edtrack--last-edit-before-original-point (edtrack point)
-  (let ((found))
+(defsubst phpinspect-edit-original-end (edit)
+  (or (caar edit) 0))
+
+(defsubst phpinspect-edit-end (edit)
+  (if edit
+      (let ((end (or (caar edit) 0))
+            (delta 0)
+            (previous-edit (cdr edit)))
+        (+ end (phpinspect-edit-delta previous-edit)))))
+
+(defsubst phpinspect-edit-delta (edit)
+  (let ((delta (or (cdar edit) 0))
+        (previous-edit edit))
+    (while (setq previous-edit (cdr previous-edit))
+      (setq delta (+ delta (cdar previous-edit))))
+    delta))
+
+(defsubst phpinspect-edtrack-original-position-at-point (track point)
+  (let ((edit (phpinspect-edtrack-edits track)))
+    (while (and edit (< point (phpinspect-edit-end edit)))
+      (setq edit (cdr edit)))
+
+    (- point (phpinspect-edit-delta edit))))
+
+(defsubst phpinspect-edtrack-current-position-at-point (track point)
+  (let ((edit (phpinspect-edtrack-edits track)))
+    (while (and edit (< point (phpinspect-edit-original-end edit)))
+      (setq edit (cdr edit)))
+
+    (+ point (phpinspect-edit-delta edit))))
+
+(defsubst phpinspect-edtrack-register-edit (track start end pre-change-length)
+  (let ((edit (phpinspect-edtrack-edits track)))
+    (while (and edit (< end (phpinspect-edit-end edit)))
+      (setq edit (cdr edit)))
+
+    (let* ((new-edit (cons (- (+ start pre-change-length) (phpinspect-edit-delta edit)) (- (- end start) pre-change-length))))
+      (if edit
+          (progn
+            (setcdr edit (cons (car edit) (cdr edit)))
+            (setcar edit new-edit))
+        (if (phpinspect-edtrack-edits track)
+            (push new-edit (cdr (last (phpinspect-edtrack-edits track))))
+          (push new-edit (phpinspect-edtrack-edits track)))))))
+
+(defsubst phpinspect-taint-start (taint)
+  (car taint))
+
+(defsubst phpinspect-taint-end (taint)
+  (cdr taint))
+
+(defsubst phpinspect-make-taint (start end)
+  (cons start end))
+
+(defsubst phpinspect-taint-overlaps-point (taint point)
+  (and (> (phpinspect-taint-end taint) point)
+       (<= (phpinspect-taint-start taint) point)))
+
+(defsubst phpinspect-taint-overlaps (taint1 taint2)
+  (or (phpinspect-taint-overlaps-point taint1 (phpinspect-taint-start taint2))
+      (phpinspect-taint-overlaps-point taint1 (phpinspect-taint-end taint2))
+      (phpinspect-taint-overlaps-point taint2 (phpinspect-taint-start taint1))
+      (phpinspect-taint-overlaps-point taint2 (phpinspect-taint-end taint1))))
+
+(defsubst phpinspect-taint-overlaps-meta (taint meta)
+  (or (phpinspect-taint-overlaps-point taint (phpinspect--meta-start meta))
+      (phpinspect-taint-overlaps-point taint (phpinspect--meta-end meta))
+      (phpinspect--meta-overlaps-point meta (phpinspect-taint-start taint))
+      (phpinspect--meta-overlaps-point meta (phpinspect-taint-end taint))))
+
+(defsubst phpinspect-edtrack-clear-taint-pool (track)
+  (setf (phpinspect-edtrack-taint-pool track) nil))
+
+(defsubst phpinspect-edtrack-clear (track)
+  (setf (phpinspect-edtrack-edits track) nil)
+  (phpinspect-edtrack-clear-taint-pool track))
+
+(defsubst phpinspect-edtrack-register-taint (track start end)
+  (let ((pool (phpinspect-edtrack-taint-pool track))
+        (idx 0)
+        (overlap-start)
+        (overlap-end)
+        (left-neighbour)
+        (taint (phpinspect-make-taint start end)))
     (catch 'break
-        (seq-doseq (edit (phpinspect-edtrack-edits edtrack))
-          (if (phpinspect-edit-before-original-point edit point)
-              (setq found edit)
-            (when found
-              (throw 'break nil)))))
-    found))
-
-(cl-defmethod phpinspect-edtrack-original-position-at-point
-  ((tracker phpinspect-edtrack) (point integer))
-  (let ((edit-before (phpinspect-edtrack--last-edit-before-point tracker point)))
-    (when edit-before
-      (setq point (- point (phpinspect-edit-delta edit-before)))))
-    point)
-
-(cl-defmethod phpinspect-edtrack-current-position-at-point
-  ((tracker phpinspect-edtrack) (point integer))
-  (let ((edit-before (phpinspect-edtrack--last-edit-before-original-point tracker point)))
-    (when edit-before
-      (setq point (+ point (phpinspect-edit-delta edit-before)))))
-
-  ;; Return
-  point)
-
-
-(defsubst phpinspect-edit-to-string (edit)
-  (format "[original-start: %d, length: %d, local-delta: %d]"
-          (phpinspect-edit-original-start edit)
-          (phpinspect-edit-length edit)
-          (phpinspect-edit-local-delta edit)))
-
+      (while pool
+        (if (phpinspect-taint-overlaps taint (car pool))
+            (progn
+              (when (< (phpinspect-taint-start (car pool)) start)
+                (setcar taint (phpinspect-taint-start (car pool))))
+              (when (> (phpinspect-taint-end (car pool)) end)
+                (setcdr taint (phpinspect-taint-end (car pool))))
+
+              (when (not overlap-start)
+                (setq overlap-start idx))
+              (setq overlap-end idx))
+
+          ;; Else
+          (when overlap-start
+            (throw 'break nil))
+
+          (when (> start (phpinspect-taint-end (car pool)))
+            (setq left-neighbour pool)
+            (throw 'break nil)))
+
+        (setq pool (cdr pool)
+              idx (+ idx 1))))
+
+    (cond (overlap-start
+           (setq pool (phpinspect-edtrack-taint-pool track))
+           (setcar (nthcdr overlap-start pool) taint)
+           (setcdr (nthcdr overlap-start pool) (nthcdr (+ 1 overlap-end) pool)))
+          (left-neighbour
+           (setcdr left-neighbour (cons taint (cdr left-neighbour))))
+          (t
+           (push taint (phpinspect-edtrack-taint-pool track))))))
 
 (provide 'phpinspect-edtrack)
+;;; phpinspect-edtrack.el ends here

phpinspect-parser.el

@@ -25,6 +25,7 @@
 
 (require 'phpinspect-tree)
 (require 'phpinspect-edtrack)
+(require 'phpinspect-bmap)
 
 (defvar phpinspect-parser-obarray (obarray-make)
   "An obarray containing symbols for all phpinspect (sub)parsers.")
@@ -523,6 +524,10 @@ parsing. Usually used in combination with
            :type phpinspect-edtrack)
   (tree nil
         :type phpinspect-tree)
+  (bmap (phpinspect-make-bmap)
+        :type phpinspect-bmap)
+  (previous-bmap nil
+                 :type phpinspect-bmap)
   (previous-tree nil
                  :type phpinspect-tree)
   (query-tree nil)
@@ -538,16 +543,17 @@ parsing. Usually used in combination with
 
 (cl-defmethod phpinspect-pctx-register-token
   ((pctx phpinspect-pctx) token start end handler)
-  (let* ((meta (phpinspect-make-meta
-                :token token
-                :handler handler
-                :whitespace-before (phpinspect-pctx-whitespace-before pctx)))
-        (node (phpinspect-tree-insert
-               (phpinspect-pctx-tree pctx) start end meta)))
-    (setf (phpinspect-meta-tree meta) node)
-    (setf (phpinspect-pctx-whitespace-before pctx) "")
-
-    meta))
+  (phpinspect-bmap-register (phpinspect-pctx-bmap pctx) start end token))
+  ;; (let* ((meta (phpinspect-make-meta
+  ;;               :token token
+  ;;               :handler handler
+  ;;               :whitespace-before (phpinspect-pctx-whitespace-before pctx)))
+  ;;       (node (phpinspect-tree-insert
+  ;;              (phpinspect-pctx-tree pctx) start end meta)))
+  ;;   (setf (phpinspect-meta-tree meta) node)
+  ;;   (setf (phpinspect-pctx-whitespace-before pctx) "")
+
+  ;;   meta))
 
 (cl-defmethod phpinspect-pctx-register-whitespace
   ((pctx phpinspect-pctx) (whitespace string))
@@ -634,6 +640,82 @@ parsing. Usually used in combination with
            ;; Return
            tokens)))))
 
+(defun phpinspect-make-bmap-parser-function (tree-type handler-list &optional delimiter-predicate)
+  "Like `phpinspect-make-parser-function', but returned function is able to reuse an already parsed tree."
+  (let ((handlers (mapcar
+                   (lambda (handler-name)
+                     (let* ((handler-name (symbol-name handler-name))
+                            (handler (intern-soft handler-name phpinspect-handler-obarray)))
+                       (if handler
+                           handler
+                         (error "No handler found by name \"%s\"" handler-name))))
+                   handler-list))
+        (delimiter-predicate (if (symbolp delimiter-predicate)
+                                 `(quote ,delimiter-predicate)
+                               delimiter-predicate)))
+    `(lambda (context buffer max-point &optional continue-condition root)
+       (with-current-buffer buffer
+         (let* ((tokens)
+                (root-start (point))
+                (bmap (phpinspect-pctx-bmap context))
+                (previous-bmap (phpinspect-pctx-previous-bmap context))
+                (edtrack (phpinspect-pctx-edtrack context))
+                (taint-iterator (when edtrack (phpinspect-edtrack-make-taint-iterator edtrack)))
+                (delimiter-predicate (when (functionp ,delimiter-predicate) ,delimiter-predicate)))
+           (phpinspect-pctx-save-whitespace context
+            (while (and (< (point) max-point)
+                        (if continue-condition (funcall continue-condition) t)
+                        (not (if delimiter-predicate
+                                 (funcall delimiter-predicate (car (last tokens)))
+                               nil)))
+              (cond ,@(mapcar
+                       (lambda (handler)
+                         `((looking-at ,(plist-get (symbol-value handler) 'regexp))
+                           (let* ((match (match-string 0))
+                                  (start-position (point))
+                                  (original-position
+                                   (when (and previous-bmap edtrack)
+                                     (phpinspect-edtrack-original-position-at-point edtrack start-position)))
+                                  (existing-meta)
+                                  (current-end-position)
+                                  (token))
+                             (when (and previous-bmap edtrack)
+                               (setq existing-meta (phpinspect-bmap-token-starting-at previous-bmap original-position))
+
+                               (when existing-meta
+                                 (setq current-end-position (phpinspect-edtrack-current-position-at-point
+                                                             edtrack (phpinspect--meta-end existing-meta)))))
+
+                             (if (and existing-meta
+                                      (not (or (phpinspect-root-p (phpinspect--meta-token existing-meta))
+                                               (phpinspect-taint-iterator-token-is-tainted-p taint-iterator existing-meta))))
+                                 (progn
+                                   (setq token (phpinspect--meta-token existing-meta))
+
+                                   ;; Re-register existing token
+                                   (let ((delta (- start-position original-position)))
+                                     (phpinspect-bmap-overlay bmap previous-bmap existing-meta delta))
+
+                                   (goto-char current-end-position))
+                               (progn
+                                 (setq token (funcall ,(symbol-function handler) match max-point))
+                                 (when token
+                                   (phpinspect-pctx-register-token context token start-position (point) ,handler))))
+                             (when token
+                               (if (null tokens)
+                                   (setq tokens (list token))
+                                 (progn
+                                   (nconc tokens (list token))))))))
+                       handlers)
+                    (t (forward-char)))))
+           (push ,tree-type tokens)
+           (when root
+             (phpinspect-pctx-register-token context tokens root-start (point) nil))
+
+           ;; Return
+           tokens)))))
+
+
 (cl-defstruct (phpinspect-parser (:constructor phpinspect-make-parser))
   (tree-keyword "root"
                 :type string
@@ -678,7 +760,7 @@ executing.")
   (or (phpinspect-parser-incremental-func parser)
       (setf (phpinspect-parser-incremental-func parser)
             (byte-compile
-             (phpinspect-make-incremental-parser-function
+             (phpinspect-make-bmap-parser-function
               (intern (concat ":" (phpinspect-parser-tree-keyword parser)))
               (phpinspect-parser-handlers parser)
               (phpinspect-parser-delimiter-predicate parser))))))

test/test-bmap.el

@@ -0,0 +1,33 @@
+
+(require 'phpinspect-bmap)
+
+(ert-deftest phpinspect-bmap-overlay ()
+  (let ((bmap (phpinspect-make-bmap))
+        (bmap2 (phpinspect-make-bmap)))
+
+    (phpinspect-bmap-register bmap 10 20 'token)
+    (phpinspect-bmap-register bmap2 20 24 'token2)
+
+    (should (phpinspect-bmap-token-starting-at bmap 10))
+
+    (phpinspect-bmap-overlay
+     bmap2 bmap (phpinspect-bmap-token-starting-at bmap 10) -3)
+
+    (should (eq 'token2 (phpinspect--meta-token
+                         (phpinspect-bmap-token-starting-at bmap2 20))))
+    (should (eq 'token (phpinspect--meta-token
+                        (phpinspect-bmap-token-starting-at bmap2 7))))
+
+    (should (phpinspect-bmap-token-meta bmap 'token))
+    (should (phpinspect-bmap-token-meta bmap2 'token2))
+    (should (phpinspect-bmap-token-meta bmap2 'token))))
+
+(ert-deftest phpinspect-bmap-nest-parent ()
+  (let ((bmap (phpinspect-make-bmap)))
+    (phpinspect-bmap-register bmap 10 20 'child)
+    (phpinspect-bmap-register bmap 5 25 'parent)
+
+    (let ((child (phpinspect-bmap-token-meta bmap 'child)))
+      (should
+       (eq 'parent (phpinspect--meta-token
+                           (phpinspect--meta-parent child)))))))

test/test-edtrack.el

@@ -1,43 +1,49 @@
 (require 'ert)
 (require 'phpinspect-edtrack)
 
+(ert-deftest phpinspect-edit-end ()
+  (let ((edit (list (cons 10 3) (cons 6 5) (cons 4 -2))))
+    (should (= 13 (phpinspect-edit-end edit)))))
+
 (ert-deftest phpinspect-edtrack-register-edit ()
   (let* ((edtrack (phpinspect-make-edtrack))
          (edit1 (phpinspect-edtrack-register-edit edtrack 5 10 10))
-         (edit2 (phpinspect-edtrack-register-edit edtrack 15 22 7))
-         (edit3 (phpinspect-edtrack-register-edit edtrack 100 200 150)))
+         (edit3 (phpinspect-edtrack-register-edit edtrack 100 200 150))
+         (edit2 (phpinspect-edtrack-register-edit edtrack 15 22 7)))
 
-    (should (= 10 (phpinspect-edit-end edit1)))
-    (should (= 22 (phpinspect-edit-end edit2)))
+    (should (equal `((255 . -50) (27 . 0) (15 . -5)) (phpinspect-edtrack-edits edtrack)))))
+    ;; (pp (phpinspect-edtrack-edits edtrack))
+    ;; (should (= 10 (phpinspect-edit-end edit1)))
+    ;; (should (= 22 (phpinspect-edit-end edit2)))
 
-    (should (= 30 (phpinspect-edtrack-original-position-at-point edtrack 25)))
-    (should (= 4 (phpinspect-edtrack-original-position-at-point edtrack 4)))
-    (should (= 260 (phpinspect-edtrack-original-position-at-point edtrack 205)))))
+    ;; (should (= 30 (phpinspect-edtrack-original-position-at-point edtrack 25)))
+    ;; (should (= 4 (phpinspect-edtrack-original-position-at-point edtrack 4)))
+    ;; (should (= 260 (phpinspect-edtrack-original-position-at-point edtrack 205)))))
 
 (ert-deftest phpinsepct-edtrack-register-multi-edits ()
-  (let* ((track (phpinspect-make-edtrack))
-         (edit (phpinspect-edtrack-register-edit track 10 20 5))
-         (edit1 (phpinspect-edtrack-register-edit track 25 30 0))
-         (edit2 (phpinspect-edtrack-register-edit track 13 20 0)))
-    (should (eq edit2 (seq-elt (phpinspect-edtrack-edits track) 0)))
-    (should (eq edit (seq-elt (phpinspect-edtrack-edits track) 1)))
-    (should (eq edit1 (seq-elt (phpinspect-edtrack-edits track) 2)))
+  (let ((track (phpinspect-make-edtrack)))
+    (phpinspect-edtrack-register-edit track 10 20 5)
+    (phpinspect-edtrack-register-edit track 25 30 0)
+    (phpinspect-edtrack-register-edit track 13 20 0)
 
     (should (= 42 (phpinspect-edtrack-current-position-at-point track 25)))))
 
 (ert-deftest phpinspect-edtrack-register-multi-edits-deletions ()
-  (let* ((track (phpinspect-make-edtrack))
-         (edit (phpinspect-edtrack-register-edit track 10 20 5))
-         (edit1 (phpinspect-edtrack-register-edit track 25 30 20))
-         (edit2 (phpinspect-edtrack-register-edit track 13 20 0)))
-    (should (eq edit2 (seq-elt (phpinspect-edtrack-edits track) 0)))
-    (should (eq edit (seq-elt (phpinspect-edtrack-edits track) 1)))
-    (should (eq edit1 (seq-elt (phpinspect-edtrack-edits track) 2)))
+  (let ((track (phpinspect-make-edtrack)))
+    (phpinspect-edtrack-register-edit track 10 20 5)
+    (phpinspect-edtrack-register-edit track 25 30 20)
+    (phpinspect-edtrack-register-edit track 13 20 0)
 
     (should (= 42 (phpinspect-edtrack-current-position-at-point track 45)))))
 
 (ert-deftest phpinspect-edtrack-register-taint ()
   (let* ((track (phpinspect-make-edtrack)))
     (phpinspect-edtrack-register-taint track 0 5)
+    (phpinspect-edtrack-register-taint track 10 20)
+    (should (equal (list (cons 0 5) (cons 10 20)) (phpinspect-edtrack-taint-pool track)))
+
+    (phpinspect-edtrack-register-taint track 3 20)
+
+    (should (equal (list (cons 0 20)) (phpinspect-edtrack-taint-pool track)))))
 
-    (should-not (phpinspect-tree-empty-p (phpinspect-edtrack-taint-pool track)))))
+    ;; (should-not (phpinspect-tree-empty-p (phpinspect-edtrack-taint-pool track)))))