WIP: Performance is terrible for large buffers
ci/woodpecker/push/woodpecker Pipeline was successful Details

WIP-incremental-parsing
Hugo Thunnissen 10 months ago
parent 92ae43fe6e
commit 0f24f7577f

@ -52,6 +52,7 @@ emacs buffer."
(cl-defmethod phpinspect-buffer-parse ((buffer phpinspect-buffer))
"Parse the PHP code in the the emacs buffer that this object is
linked with."
(phpinspect-buffer-propagate-taints buffer)
(with-current-buffer (phpinspect-buffer-buffer buffer)
(let* ((tree (phpinspect-make-tree :start (point-min)
:end (+ 1 (point-max))))
@ -67,30 +68,37 @@ linked with."
(setq tree (seq-elt (phpinspect-tree-children tree) 0))
(setf (phpinspect-tree-parent tree) nil)
(setf (phpinspect-buffer-tree buffer) tree)
(setf (phpinspect-edtrack-edits (phpinspect-buffer-edit-tracker buffer))
(phpinspect-make-ll))
(phpinspect-edtrack-clear (phpinspect-buffer-edit-tracker buffer))
;; return
parsed)))))
(cl-defmethod phpinspect-buffer-reparse ((buffer phpinspect-buffer))
(setf (phpinspect-buffer-tree buffer) (phpinspect-make-tree))
(phpinspect-buffer-parse buffer))
(defsubst phpinspect-buffer-parse-tree (buffer)
(phpinspect-buffer-parse buffer)
(phpinspect-buffer-tree buffer))
(cl-defmethod phpinspect-buffer-register-edit
((buffer phpinspect-buffer) (start integer) (end integer) (pre-change-length integer))
(let* ((edit
(phpinspect-edtrack-register-edit
(phpinspect-buffer-edit-tracker buffer) start end pre-change-length))
(region (phpinspect-make-region (phpinspect-edit-original-start edit)
(phpinspect-edit-original-end edit)))
(tainted (phpinspect-tree-traverse-overlapping
(phpinspect-buffer-tree buffer) region)))
(dolist (meta tainted)
(phpinspect-tree-traverse (node (phpinspect-meta-tree meta))
(when (phpinspect-tree-overlaps node region)
(setf (phpinspect-meta-tainted (phpinspect-tree-value node)) t))))))
(phpinspect-edtrack-register-edit
(phpinspect-buffer-edit-tracker buffer) start end pre-change-length))
(cl-defmethod phpinspect-buffer-propagate-taints ((buffer phpinspect-buffer))
(let ((tracker (phpinspect-buffer-edit-tracker buffer)))
(when (phpinspect-edtrack-has-taints-p tracker)
(seq-doseq (taint (phpinspect-tree-children (phpinspect-edtrack-taint-pool tracker)))
(let* ((region (phpinspect-make-region (phpinspect-tree-start taint)
(phpinspect-tree-end taint)))
(tainted (phpinspect-tree-traverse-overlapping
(phpinspect-buffer-tree buffer) region)))
(dolist (meta tainted)
(setf (phpinspect-meta-tainted meta) t))))
(phpinspect-edtrack-clear-taints tracker))))
(cl-defmethod phpinspect-buffer-tokens-enclosing-point ((buffer phpinspect-buffer) point)
(phpinspect-tree-traverse-overlapping (phpinspect-buffer-tree buffer) point))

@ -2,7 +2,9 @@
(cl-defstruct (phpinspect-edtrack (:constructor phpinspect-make-edtrack))
(edits (phpinspect-make-ll)
:documentation "Sorted list of edits in buffer"))
:documentation "Sorted list of edits in buffer")
(taint-pool (phpinspect-make-tree :grow-root t)
:documentation "Non overlapping pool of tainted buffer regions"))
(cl-defstruct (phpinspect-edit (:constructor phpinspect-make-edit))
@ -68,10 +70,34 @@
(phpinspect-region-overlaps-point region (phpinspect-edit-original-start edit))
(phpinspect-region-overlaps-point region (- (phpinspect-edit-original-end edit) 1)))))
(defsubst phpinspect-edtrack-clear-taints (tracker)
(setf (phpinspect-edtrack-taint-pool tracker) (phpinspect-make-tree :grow-root t)))
(defsubst phpinspect-edtrack-clear (tracker)
(phpinspect-edtrack-clear-taints tracker)
(setf (phpinspect-edtrack-edits tracker) (phpinspect-make-ll)))
(defsubst phpinspect-edtrack-has-taints-p (tracker)
(not (phpinspect-tree-empty-p (phpinspect-edtrack-taint-pool tracker))))
(defsubst phpinspect-edtrack-register-taint (tracker start end)
(let* ((pool (phpinspect-edtrack-taint-pool tracker))
(overlappers (phpinspect-tree-find-overlapping-children pool start end)))
(when overlappers
(seq-doseq (overlapper overlappers)
(when (> (phpinspect-tree-end overlapper) end)
(setq end (phpinspect-tree-end overlapper)))
(when (< (phpinspect-tree-start overlapper) start)
(setq start (phpinspect-tree-start overlapper))))
(phpinspect-slice-detach overlappers))
(phpinspect-tree-insert pool start end 'edited)))
(cl-defmethod phpinspect-edtrack-register-edit
((tracker phpinspect-edtrack) (start integer) (end integer) (pre-change-length integer))
(let* ((overlap-test (lambda (edit) (phpinspect-edit-overlaps edit start end)))
(edits (phpinspect-edtrack-edits tracker))
(let* ((edits (phpinspect-edtrack-edits tracker))
(first-overlap)
(last-overlap)
(edit-before)
@ -101,6 +127,9 @@
(phpinspect-ll-insert-right (phpinspect-edit-link edit-before) new-edit)
(phpinspect-ll-push new-edit edits))
(phpinspect-edtrack-register-taint
tracker (phpinspect-edit-original-start new-edit) (phpinspect-edit-original-end new-edit))
;; Return
new-edit))

@ -35,55 +35,50 @@
(goto-char point)
(insert data)))
(defun phpinspect-add-use (fqn buffer &optional namespace-token)
(cl-defmethod phpinspect-namespace-body-start ((tree phpinspect-tree))
(if (not (seq-emtpy-p (phpinspect-tree-children tree)))
(let ((block (seq-elt (phpinspect-tree-children tree) 2)))
(if (phpinspect-block-p (phpinspect-tree-meta-token block))
(phpinspect-tree-start block)
(phpinspect-tree-end (seq-elt (phpinspect-tree-children tree) 1))))
0))
(defun phpinspect-add-use (fqn buffer &optional namespace-tree)
"Add use statement for FQN to BUFFER.
If NAMESPACE-TOKEN is non-nil, it is assumed to be a token that
was parsed from BUFFER and its location will be used to find a
If NAMESPACE-TREE is non-nil, it is assumed to be an instance of
`phpinspect-tree' containing metadata of a namespace token that
was parsed from BUFFER. Its location will be used to find a
buffer position to insert the use statement at."
(when (string-match "^\\\\" fqn)
(setq fqn (string-trim-left fqn "\\\\")))
(if namespace-token
(let* ((region (gethash
namespace-token (phpinspect-buffer-location-map buffer)))
(existing-use (seq-find #'phpinspect-use-p
(phpinspect-namespace-body namespace-token)))
(if namespace-tree
(let* ((existing-use (seq-find (phpinspect-tree-meta-token-filter #'phpinspect-use-p)
(phpinspect-namespace-body namespace-tree)))
(namespace-block (phpinspect-namespace-block namespace-token)))
(if existing-use
(phpinspect-insert-at-point
(phpinspect-region-start
(phpinspect-buffer-token-location buffer existing-use))
(format "use %s;%c" fqn ?\n))
(if namespace-block
(phpinspect-insert-at-point
(+ 1 (phpinspect-region-start
(phpinspect-buffer-token-location buffer namespace-block)))
(format "%c%cuse %s;%c" ?\n ?\n fqn ?\n))
(phpinspect-insert-at-point
(phpinspect-region-end
(phpinspect-buffer-token-location
buffer (seq-find #'phpinspect-terminator-p namespace-token)))
(format "%c%cuse %s;%c" ?\n ?\n fqn ?\n)))))
(phpinspect-tree-start existing-use) (format "use %s;%c" fqn ?\n))
(phpinspect-insert-at-point
(+ 1 (phpinspect-namespace-body-start namespace-tree))
(format "%c%cuse %s;%c" ?\n ?\n fqn ?\n))))
;; else
(let ((existing-use (seq-find #'phpinspect-use-p
(let ((existing-use (seq-find (phpinspect-tree-meta-token-filter #'phpinspect-use-p)
(phpinspect-buffer-tree buffer))))
(if existing-use
(phpinspect-insert-at-point
(phpinspect-region-start
(phpinspect-buffer-token-location buffer existing-use))
(phpinspect-tree-start existing-use)
(format "use %s;%c" fqn ?\n))
(let ((first-token (cadr (phpinspect-buffer-tree buffer))))
(if (and (phpinspect-word-p first-token)
(string= "declare" (cadr first-token)))
(let ((first-token (seq-elt (phpinspect-tree-children (phpinspect-buffer-tree buffer)) 0)))
(if (and (phpinspect-word-p (phpinspect-tree-meta-token first-token))
(string= "declare" (cadr (phpinspect-tree-meta-token first-token))))
(phpinspect-insert-at-point
(phpinspect-region-end
(phpinspect-buffer-token-location
buffer (seq-find #'phpinspect-terminator-p (phpinspect-buffer-tree buffer))))
(format "%c%cuse %s;%c" ?\n ?\n fqn ?\n))
(phpinspect-tree-end (seq-find (phpinspect-tree-meta-token-filter #'phpinspect-terminator-p)
(phpinspect-tree-children (phpinspect-buffer-tree buffer))))
(format "%c%cuse %s;%c" ?\n ?\n fqn ?\n))
(phpinspect-insert-at-point
(phpinspect-region-start
(phpinspect-buffer-token-location buffer first-token))
(phpinspect-tree-start first-token)
(format "%c%cuse %s;%c%c" ?\n ?\n fqn ?\n ?\n))))))))
(defun phpinspect-add-use-interactive (typename buffer project &optional namespace-token)
@ -109,10 +104,9 @@ buffer position to insert the use statement at."
that there are import (\"use\") statements for them."
(interactive)
(if phpinspect-current-buffer
(let* ((tree (phpinspect-buffer-parse phpinspect-current-buffer))
(location-map (phpinspect-buffer-location-map phpinspect-current-buffer))
(index (phpinspect--index-tokens
tree nil (lambda (token) (gethash token location-map))))
(phpinspect-buffer-parse phpinspect-current-buffer)
(let* ((tree (phpinspect-buffer-tree phpinspect-current-buffer))
(index (phpinspect--index-tokens tree))
(classes (alist-get 'classes index))
(imports (alist-get 'imports index))
(project (phpinspect--cache-get-project-create
@ -121,12 +115,10 @@ that there are import (\"use\") statements for them."
(dolist (class classes)
(let* ((class-imports (alist-get 'imports class))
(used-types (alist-get 'used-types class))
(region (alist-get 'location class)))
(meta (alist-get 'token-metadata class)))
(dolist (type used-types)
(let ((namespace
(seq-find #'phpinspect-namespace-p
(phpinspect-buffer-tokens-enclosing-point
phpinspect-current-buffer (phpinspect-region-start region)))))
(phpinspect-tree-find-parent-meta-matching tree #'phpinspect-namespace-p)))
;; Add use statements for types that aren't imported.
(unless (or (or (alist-get type class-imports)
@ -140,6 +132,7 @@ that there are import (\"use\") statements for them."
(phpinspect-project-autoload project))))
(phpinspect-add-use-interactive
type phpinspect-current-buffer project namespace)
;;;; FIXED?
;; Buffer has been modified by adding type, update tree +
;; location map. This is not optimal but will have to do until
;; partial parsing is implemented.

@ -525,9 +525,17 @@ parsing. Usually used in combination with
:type phpinspect-tree)
(previous-tree nil
:type phpinspect-tree)
(query-tree nil)
(whitespace-before ""
:type string))
(defsubst phpinspect-pctx-find-existing-node-at-point (ctx point)
(let ((query-tree (or (phpinspect-pctx-query-tree ctx)
(phpinspect-pctx-previous-tree ctx))))
(when query-tree
(setf (phpinspect-pctx-query-tree ctx)
(phpinspect-tree-find-next-relative-starting-at query-tree point)))))
(cl-defmethod phpinspect-pctx-register-token
((pctx phpinspect-pctx) token start end handler)
(let* ((meta (phpinspect-make-meta
@ -585,7 +593,9 @@ parsing. Usually used in combination with
(current-end-position)
(token))
(unless (not previous-tree)
(setq existing-node (phpinspect-tree-find-node-starting-at previous-tree original-position))
(message "Searching for existing node at point %d" (point))
(setq existing-node (phpinspect-pctx-find-existing-node-at-point
context original-position))
(when existing-node
(setq existing-meta (phpinspect-tree-value existing-node)
current-end-position (phpinspect-edtrack-current-position-at-point
@ -599,7 +609,10 @@ parsing. Usually used in combination with
(setq token (phpinspect-meta-token existing-meta))
;; Alter regions to current token position in buffer
(let ((delta (- start-position original-position)))
(phpinspect-tree-shift existing-node delta))
(unless (= 0 delta)
(message "Shifting tree with delta %d" delta)
(message "point: %d, start position: %d" (point) start-position)
(phpinspect-tree-shift existing-node delta)))
(goto-char current-end-position)
;; Insert existing token into new tree

@ -40,6 +40,8 @@ being said, not doing so will not limit the trees
functionalities."
(parent nil
:type phpinspect-tree)
(grow-root nil
:type boolean)
(children (phpinspect-make-ll)
:type phpinspect-llnode)
(start 0
@ -142,24 +144,24 @@ belongs to. Return resulting linked list."
(setf (phpinspect-slice-start slice) start)
(setf (phpinspect-slice-end slice) end)))
(if (eq start end)
start
(unless (eq start end)
(when right-neighbour
(setf (phpinspect-llnode-left right-neighbour) left-neighbour))
(setf (phpinspect-llnode-right left-neighbour) right-neighbour)
(setf (phpinspect-llnode-left start) nil)
(setf (phpinspect-llnode-right end) nil)
(setf (phpinspect-llnode-right end) nil))
;; Fix broken references in old link-map and create separate link-map for
;; the new detached list.
(let ((list start)
(link-map (make-hash-table :test #'eq :size 100 :rehash-size 400)))
(while list
(phpinspect-ll-unregister-link list)
(setf (phpinspect-llnode-link-map list) link-map)
(phpinspect-ll-register-link list)
(setq list (phpinspect-llnode-right list))))
;; Fix broken references in old link-map and create separate link-map for
;; the new detached list.
(let ((list start)
(link-map (make-hash-table :test #'eq :size 100 :rehash-size 400)))
(while list
(phpinspect-ll-unregister-link list)
(setf (phpinspect-llnode-link-map list) link-map)
(phpinspect-ll-register-link list)
(setq list (phpinspect-llnode-right list)))
start)))
@ -443,6 +445,38 @@ belongs to. Return resulting linked list."
(and (= 0 (phpinspect-tree-start tree))
(= 0 (phpinspect-tree-end tree))))
(cl-defmethod phpinspect-tree-find-next-relative-starting-at ((tree phpinspect-tree) (point integer))
(when (< point (phpinspect-tree-start tree))
(error "Can't find next relative when point is before tree start"))
(let ((parent (phpinspect-tree-parent tree))
(found?))
(catch 'found
;; First check own children
(when (and (> (phpinspect-tree-end tree) point)
(setq found? (phpinspect-tree-find-node-starting-at tree point)))
(throw 'found found?))
(while parent
(when (> (phpinspect-tree-end parent) point)
;; Check siblings after
(let ((parent-link (phpinspect-ll-link (phpinspect-tree-children parent) tree)))
(seq-doseq (sibling parent-link)
(when (setq found? (phpinspect-tree-find-node-starting-at sibling point))
(throw 'found found?)))))
(setq tree parent)
(setq parent (phpinspect-tree-parent parent))))))
(cl-defmethod phpinspect-tree-envelop ((tree phpinspect-tree) (node phpinspect-tree))
(when (< (phpinspect-tree-start node) (phpinspect-tree-start tree))
(setf (phpinspect-tree-start tree) (phpinspect-tree-start node)))
(when (> (phpinspect-tree-end node) (phpinspect-tree-end tree))
(setf (phpinspect-tree-end tree) (phpinspect-tree-end node))))
(cl-defmethod phpinspect-tree-find-last-child-before-point ((tree phpinspect-tree) (point integer))
(catch 'found
(seq-doseq (child (seq-reverse (seq-into (phpinspect-tree-children tree) 'slice)))
@ -453,6 +487,9 @@ belongs to. Return resulting linked list."
"Insert a new NODE into TREE.
Returns the newly inserted node."
(when (phpinspect-tree-grow-root tree)
(phpinspect-tree-envelop tree node))
(cond ((phpinspect-tree-empty-p tree)
(phpinspect-tree-switch-attributes node tree)
@ -466,10 +503,9 @@ Returns the newly inserted node."
(overlap-count (seq-length overlappers)))
(if overlappers
(cond
((= 1 overlap-count)
(phpinspect-tree-insert-node (seq-elt overlappers 0)
node))
((< 1 overlap-count)
((or (< 1 overlap-count)
(and (= 1 overlap-count)
(phpinspect-tree-encloses node (seq-elt overlappers 0))))
;; There are multiple overlapping children. They need to all
;; fit within node, or the hierarchy is broken.
(let ((enclosed
@ -477,47 +513,46 @@ Returns the newly inserted node."
(lambda (child) (phpinspect-tree-encloses node child))
overlappers))
(insert-after-link))
(unless (= (seq-length enclosed) overlap-count)
(seq-doseq (lap overlappers)
(message "overlaps: %s (%d,%d) with %s (%d,%d)"
(phpinspect-meta-token (phpinspect-tree-value lap))
(phpinspect-tree-start lap)
(phpinspect-tree-end lap)
(phpinspect-meta-token (phpinspect-tree-value node))
(phpinspect-tree-start node)
(phpinspect-tree-end node)))
(throw 'phpinspect-tree-conflict
"Node overlaps multiple children, but does not enclose them all"))
;; Find the list link that the first enclosed node is attached to.
(setq insert-after-link (phpinspect-llnode-left
(phpinspect-slice-start enclosed)))
;; Remove enclosed nodes from parent
(setq enclosed (phpinspect-slice-detach enclosed))
(if insert-after-link
;; Insert new node into old enclosed node position
(phpinspect-ll-insert-right insert-after-link node)
;; If there is nothing to the left of the enclosed regions,
;; we can safely push to the tree's children
(phpinspect-ll-push node (phpinspect-tree-children tree)))
(setf (phpinspect-tree-parent node) tree)
(seq-doseq (child enclosed)
(setf (phpinspect-tree-parent child) node))
(setf (phpinspect-tree-children node) enclosed))))
(setf (phpinspect-tree-children node) enclosed)))
((= 1 overlap-count)
(phpinspect-tree-insert-node (seq-elt overlappers 0)
node))
)
;; ELSE: No overlap, node can safely be added as child
(setf (phpinspect-tree-parent node) tree)
(let* ((right-neighbour (phpinspect-tree-children tree))
(right-neighbour-value
(seq-find (lambda (child) (< (phpinspect-tree-end child)
(phpinspect-tree-start node)))
right-neighbour)))
(when right-neighbour-value
(setq right-neighbour (phpinspect-ll-link
right-neighbour
right-neighbour-value)))
(if (phpinspect-llnode-left right-neighbour)
(phpinspect-ll-insert-left right-neighbour node)
(phpinspect-ll-push node right-neighbour)))))
(let* ((left-neighbour (phpinspect-tree-children tree))
(left-neighbour-value
(phpinspect-tree-find-last-child-before-point tree (phpinspect-tree-start node))))
(if left-neighbour-value
(progn
(setq left-neighbour (phpinspect-ll-link left-neighbour left-neighbour-value))
(phpinspect-ll-insert-right left-neighbour node))
(phpinspect-ll-push node left-neighbour)))))
;; Return
node)
@ -545,13 +580,7 @@ Returns the newly inserted node."
;; Return tree, as this is the node that value of node has been
;; stored in.
tree))))
(t (message "parent: %s" (when (phpinspect-tree-value tree)
(phpinspect-meta-token (phpinspect-tree-value tree))))
(message "perspective child: %s"
(when (phpinspect-tree-value node)
(phpinspect-meta-token (phpinspect-tree-value node))))
(throw 'phpinspect-tree-conflict
(t (throw 'phpinspect-tree-conflict
(format "Tree does not enclose or get enclosed. \nTree: (%d,%d,%s) \n\nPerspective child: (%d,%d,%s)"
(phpinspect-tree-start tree)
(phpinspect-tree-end tree)
@ -637,16 +666,22 @@ with its value as argument."
(when children
(seq-doseq (child children)
(phpinspect-tree-widen-after-point child point)))))
(phpinspect-tree-widen-after-point child point delta)))))
(cl-defmethod phpinspect-tree-find-node-starting-at ((tree phpinspect-tree) (point integer))
(if (= (phpinspect-tree-start tree) point)
tree
(catch 'found
(seq-doseq (child (phpinspect-tree-children tree))
(when (phpinspect-tree-overlaps tree point)
(let ((found? (phpinspect-tree-find-node-starting-at child point)))
(when found? (throw 'found found?))))))))
(let ((overlapped))
(seq-doseq (child (phpinspect-tree-children tree))
(if (phpinspect-tree-overlaps tree point)
(progn
(setq overlapped t)
(let ((found? (phpinspect-tree-find-node-starting-at child point)))
(when found? (throw 'found found?))))
;; Stop iterating when overlap stops
(when overlapped (throw 'found nil))))))))
(cl-defmethod phpinspect-tree-width ((tree phpinspect-tree))
(- (phpinspect-tree-start tree) (phpinspect-tree-end tree)))

@ -76,6 +76,7 @@
;; "Deletes" first curly brace of __construct function block
(phpinspect-buffer-register-edit buffer 1036 1036 1)
(phpinspect-buffer-propagate-taints buffer)
(let* ((region (phpinspect-make-region 1036 1037))
(tainted
@ -145,4 +146,6 @@
(setq parsed (phpinspect-buffer-parse buffer))
(should parsed)
(setq hello2 (car (phpinspect-buffer-tokens-enclosing-point buffer 18)))
(should (eq hello hello2)))))
(should (eq hello hello2))
(setq parsed (phpinspect-buffer-parse-tree buffer)))))

@ -25,7 +25,7 @@
(should (= 42 (phpinspect-edtrack-current-position-at-point track 25)))))
(ert-deftest phpinsepct-edtrack-register-multi-edits-deletions ()
(ert-deftest phpinspect-edtrack-register-multi-edits-deletions ()
(let* ((track (phpinspect-make-edtrack))
(edit (phpinspect-edtrack-register-edit track 10 20 5))
(edit1 (phpinspect-edtrack-register-edit track 25 30 20))
@ -35,3 +35,9 @@
(should (eq edit1 (seq-elt (phpinspect-edtrack-edits track) 2)))
(should (= 42 (phpinspect-edtrack-current-position-at-point track 45)))))
(ert-deftest phpinspect-edtrack-register-taint ()
(let* ((track (phpinspect-make-edtrack)))
(phpinspect-edtrack-register-taint track 0 5)
(should-not (phpinspect-tree-empty-p (phpinspect-edtrack-taint-pool track)))))

Loading…
Cancel
Save