WIP: Index types in the background using collaborative threading.

Created queue for types to be indexed in separate thread and tried to limit
lock-ups as much as possible by using idle timers.

phpinspect-cache.el

@@ -56,7 +56,7 @@ then returned.")
   ((cache phpinspect--cache) (project-root string))
   (or (phpinspect--cache-getproject cache project-root)
       (puthash project-root
-               (phpinspect--make-project-cache)
+               (phpinspect--make-project-cache :root project-root)
                (phpinspect--cache-projects cache))))
 
 (provide 'phpinspect-cache)

phpinspect-index.el

@@ -338,12 +338,6 @@ namespace if not provided"
   ;; TODO: Implement function indexation
   )
 
-;; (defun phpinspect--get-or-create-index-for-class-file (class-fqn)
-;;   (phpinspect--log "Getting or creating index for %s" class-fqn)
-;;   (phpinspect-get-or-create-cached-project-class
-;;    (phpinspect-project-root)
-;;    class-fqn))
-
 (defun phpinspect-index-file (file-name)
   (phpinspect--index-tokens (phpinspect-parse-file file-name)))
 
@@ -354,34 +348,6 @@ namespace if not provided"
                     project-root)))
       (phpinspect--project-get-class-create project class-fqn))))
 
-    ;; (let ((existing-index (phpinspect-get-cached-project-class
-    ;;                        project-root
-    ;;                        class-fqn)))
-    ;;   (or
-    ;;    existing-index
-    ;;    (progn
-    ;;      (let* ((class-file (phpinspect-class-filepath class-fqn))
-    ;;             (visited-buffer (when class-file (find-buffer-visiting class-file)))
-    ;;             (new-index))
-
-    ;;        (phpinspect--log "No existing index for FQN: %s" class-fqn)
-    ;;        (phpinspect--log "filepath: %s" class-file)
-    ;;        (when class-file
-    ;;          (if visited-buffer
-    ;;              (setq new-index (with-current-buffer visited-buffer
-    ;;                                (phpinspect--index-current-buffer)))
-    ;;            (setq new-index (phpinspect-index-file class-file)))
-    ;;          (dolist (class (alist-get 'classes new-index))
-    ;;            (when class
-    ;;              (phpinspect-cache-project-class
-    ;;               project-root
-    ;;               (cdr class))))
-    ;;          (alist-get class-fqn (alist-get 'classes new-index)
-    ;;                     nil
-    ;;                     nil
-    ;;                     #'phpinspect--type=))))))))
-
-
 (defun phpinspect--index-current-buffer ()
   (phpinspect--index-tokens (phpinspect-parse-current-buffer)))
 
@@ -389,15 +355,167 @@ namespace if not provided"
   "Index a PHP file for classes and the methods they have"
   (phpinspect--index-tokens (phpinspect-parse-current-buffer)))
 
-;; (defun phpinspect--get-variables-for-class (buffer-classes class &optional static)
+(cl-defstruct (phpinspect--queue-item
-;;   (let ((class-index (or (assoc-default class buffer-classes #'phpinspect--type=)
+               (:constructor phpinspect--make-queue-item))
-;;                          (phpinspect--get-or-create-index-for-class-file class))))
+  (next nil
-;;     (when class-index
+        :type phpinspect--queue-item
-;;       (if static
+        :documentation
-;;           (append (alist-get 'static-variables class-index)
+        "The next item in the queue")
-;;                   (alist-get 'constants class-index))
+  (thing nil
-;;         (alist-get 'variables class-index)))))
+         :type any
-
+         :documentation
+         "The thing stored in the queue")
+  (previous nil
+            :type phpinspect--queue-item
+            :documentation
+            "The previous item in the queue"))
+
+(cl-defmethod phpinspect--queue-last ((item phpinspect--queue-item))
+  (if (phpinspect--queue-item-next item)
+      (phpinspect--queue-last (phpinspect--queue-item-next item))
+    item))
+
+(cl-defmethod phpinspect--queue-first ((item phpinspect--queue-item))
+  (if (phpinspect--queue-item-previous item)
+      (phpinspect--queue-first (phpinspect--queue-item-previous item))
+    item))
+
+(cl-defmethod phpinspect--queue-enqueue ((item phpinspect--queue-item) thing)
+  (let ((last (phpinspect--queue-last item)))
+    (if (not (phpinspect--queue-item-thing last))
+        (setf (phpinspect--queue-item-thing last) thing)
+  (setf (phpinspect--queue-item-next last)
+        (phpinspect--make-queue-item :previous last :thing thing)))))
+
+(cl-defmethod phpinspect--queue-dequeue ((item phpinspect--queue-item))
+  (let* ((first (phpinspect--queue-first item))
+         (thing (phpinspect--queue-item-thing first))
+         (next (phpinspect--queue-item-next first)))
+    (when next (setf (phpinspect--queue-item-previous next) nil))
+    (cond ((and (eq item first) (not next))
+           (setf (phpinspect--queue-item-thing item)
+                 nil))
+          ((eq item first)
+
+           (setf (phpinspect--queue-item-thing item)
+                 (phpinspect--queue-item-thing next))
+           (setf (phpinspect--queue-item-next item)
+                 (phpinspect--queue-item-next next))))
+    thing))
+
+(cl-defmethod phpinspect--queue-find
+  ((item phpinspect--queue-item) thing comparison-func)
+  (setq item (phpinspect--queue-first item))
+
+  (let ((found))
+    (while (and item (not found))
+      (when (and (phpinspect--queue-item-thing item)
+                 (funcall comparison-func thing (phpinspect--queue-item-thing item)))
+        (setq found item))
+
+      (setq item (phpinspect--queue-item-next item)))
+
+    found))
+
+(cl-defmethod phpinspect--queue-enqueue-noduplicate
+  ((item phpinspect--queue-item) thing comparison-func)
+
+  (when (not (phpinspect--queue-find item thing comparison-func))
+    (phpinspect--queue-enqueue item thing)))
+
+(defvar phpinspect--index-queue (phpinspect--make-queue-item)
+  "Queue with indexation tasks. Each task is a list, the car of
+  which is a project directory path and the cadr of which is an
+  instance of `phpinspect--type`.")
+
+(defvar phpinspect--index-thread nil
+  "Thread that executes index tasks from `phpinspect--index-queue`.")
+
+(defvar phpinspect--index-thread-running nil
+  "Thread that executes index tasks from `phpinspect--index-queue`.")
+
+(defun phpinspect--index-thread-function ()
+  (while phpinspect--index-thread-running
+    (let* ((task (phpinspect--queue-dequeue phpinspect--index-queue))
+           (mx (make-mutex))
+           (continue (make-condition-variable mx))
+           (skip-pause))
+      (when task
+        (phpinspect--log "Indexing class %s for project in %s from index thread"
+                         (phpinspect--index-task-type task)
+                         (phpinspect--index-task-project-root task))
+
+        (let ((project (phpinspect--cache-get-project-create
+                         (phpinspect--get-or-create-global-cache)
+                         (phpinspect--index-task-project-root task)))
+              (is-native-type (phpinspect--type-is-native
+                               (phpinspect--index-task-type task))))
+          (if is-native-type
+              (progn
+                (phpinspect--log "Skipping indexation of native type %s"
+                                 (phpinspect--index-task-type task))
+                (setq skip-pause t))
+            (let ((type-index (phpinspect--index-type
+                                 project
+                                 (phpinspect--index-task-type task))))
+              (when type-index (phpinspect--project-add-class project type-index))))))
+
+          (unless skip-pause
+            (phpinspect--index-thread-pause 1 mx continue))
+          (setq skip-pause nil)))
+  (phpinspect--log "Index thread exiting")
+  (message "phpinspect index thread exited"))
+
+(defun phpinspect--index-thread-pause (pause-time mx continue)
+  (phpinspect--log "Index thead is paused for %d seconds" pause-time)
+  (run-with-idle-timer
+   pause-time
+   nil
+   (lambda () (with-mutex mx (condition-notify continue))))
+  (with-mutex mx (condition-wait continue))
+  (phpinspect--log "Index thread continuing"))
+
+(defun phpinspect--ensure-index-thread ()
+  (interactive)
+  (when (or (not phpinspect--index-thread)
+            (not (thread-alive-p phpinspect--index-thread)))
+    (setq phpinspect--index-thread-running t)
+    (setq phpinspect--index-thread
+          (make-thread #'phpinspect--index-thread-function "phpinspect-index-thread"))))
+
+(defun phpinspect--stop-index-thread ()
+  (interactive)
+  (setq phpinspect--index-thread-running nil))
+
+(defalias 'phpinspect--index-task-project-root #'car)
+(defalias 'phpinspect--index-task-type #'cadr)
+
+(defun phpinspect--index-task= (task1 task2)
+  (and (phpinspect--type= (phpinspect--index-task-type task1)
+                          (phpinspect--index-task-type task2))
+       (string= (phpinspect--index-task-project-root task1)
+                (phpinspect--index-task-project-root task2))))
+
+(defsubst phpinspect--make-index-task (project-root type)
+  (list project-root type))
+
+(cl-defmethod phpinspect--index-type ((project phpinspect--project)
+                                      (type phpinspect--type))
+  (let* ((class-file (with-temp-buffer
+                       (cd (phpinspect--project-root project))
+                       (phpinspect-type-filepath type)))
+         (visited-buffer (when class-file (find-buffer-visiting class-file)))
+         (new-index)
+         (class-index))
+    (when class-file
+      (if visited-buffer
+          (setq new-index (with-current-buffer visited-buffer
+                            (phpinspect--index-current-buffer)))
+        (setq new-index (phpinspect-index-file class-file)))
+            (alist-get type (alist-get 'classes new-index)
+                       nil
+                       nil
+                       #'phpinspect--type=))))
 
 (provide 'phpinspect-index)
 ;;; phpinspect-index.el ends here

phpinspect-project.el

@@ -31,12 +31,26 @@
                :type hash-table
                :documentation
                "A `hash-table` that contains all of the currently
-indexed classes in the project"))
+indexed classes in the project")
+  (root nil
+        :type string
+        :documentation
+        "The root directory of this project"))
 
 (cl-defgeneric phpinspect--project-add-class
     ((project phpinspect--project) (class (head phpinspect--indexed-class)))
   "Add an indexed CLASS to PROJECT.")
 
+(cl-defmethod phpinspect--project-add-return-types-to-index-queueue
+  ((project phpinspect--project) methods)
+  (dolist (method methods)
+    (when (not (phpinspect--project-get-class project (phpinspect--function-return-type method)))
+      (phpinspect--queue-enqueue-noduplicate phpinspect--index-queue
+                                             (phpinspect--make-index-task
+                                              (phpinspect--project-root project)
+                                              (phpinspect--function-return-type method))
+                                             #'phpinspect--index-task=))))
+
 (cl-defmethod phpinspect--project-add-class
   ((project phpinspect--project) (indexed-class (head phpinspect--indexed-class)))
   (let* ((class-name (phpinspect--type-name-symbol
@@ -44,10 +58,17 @@ indexed classes in the project"))
          (existing-class (gethash class-name
                                   (phpinspect--project-class-index project))))
     (if existing-class
-        (phpinspect--class-set-index existing-class indexed-class)
+        (progn
+          (phpinspect--class-set-index existing-class indexed-class)
+          (phpinspect--project-add-return-types-to-index-queueue
+           project
+           (phpinspect--class-get-method-list existing-class)))
       (let ((new-class (phpinspect--make-class-generated :project project)))
         (phpinspect--class-set-index new-class indexed-class)
-        (puthash class-name new-class (phpinspect--project-class-index project))))))
+        (puthash class-name new-class (phpinspect--project-class-index project))
+        (phpinspect--project-add-return-types-to-index-queueue
+         project
+         (phpinspect--class-get-method-list new-class))))))
 
 (cl-defgeneric phpinspect--project-get-class
     ((project phpinspect--project) (class-fqn phpinspect--type))
@@ -61,28 +82,15 @@ indexed classes in the project"))
       (puthash (phpinspect--type-name-symbol class-fqn)
                class
                (phpinspect--project-class-index project))
-
+      (phpinspect--queue-enqueue-noduplicate
-      (let* ((class-file (phpinspect-class-filepath class-fqn))
+       phpinspect--index-queue
-             (visited-buffer (when class-file (find-buffer-visiting class-file)))
+       (phpinspect--make-index-task (phpinspect--project-root project)
-             (new-index)
+                                    class-fqn)
-             (class-index))
+       #'phpinspect--index-task=))
-
-        (phpinspect--log "No existing index for FQN: %s" class-fqn)
-        (phpinspect--log "filepath: %s" class-file)
-        (when class-file
-          (if visited-buffer
-              (setq new-index (with-current-buffer visited-buffer
-                                (phpinspect--index-current-buffer)))
-            (setq new-index (phpinspect-index-file class-file)))
-          (setq class-index
-                (alist-get class-fqn (alist-get 'classes new-index)
-                           nil
-                           nil
-                           #'phpinspect--type=))
-          (when class-index
-            (phpinspect--class-set-index class class-index)))))
     class))
 
+(defalias 'phpinspect--project-add-class-if-missing #'phpinspect--project-get-class-create)
+
 (cl-defmethod phpinspect--project-get-class
   ((project phpinspect--project) (class-fqn phpinspect--type))
   (gethash (phpinspect--type-name-symbol class-fqn)

phpinspect-type.el

@@ -49,11 +49,11 @@
   `(phpinspect--make-type-generated
     ,@(phpinspect--wrap-plist-name-in-symbol property-list)))
 
-(defun phpinspect--make-types (&rest type-names)
+(defun phpinspect--make-types (type-names)
   (mapcar (lambda (name) (phpinspect--make-type :name name))
           type-names))
 
-(defconst phpinspect-native-types
+(defconst phpinspect-native-typenames
   ;; self, parent and resource are not valid type name.
   ;; see https://www.php.net/manual/ja/language.types.declarations.php
   ;;;
@@ -61,6 +61,10 @@
   ;; list of type names that we should not attempt to resolve relatively.
   '("array" "bool" "callable" "float" "int" "iterable" "mixed" "object" "string" "void" "self" "static" "this"))
 
+(defconst phpinspect-native-types
+  (phpinspect--make-types (mapcar (lambda (name) (concat "\\" name))
+                                  phpinspect-native-typenames)))
+
 (defvar phpinspect-collection-types
   (phpinspect--make-types '("\\array" "\\iterable" "\\SplObjectCollection" "\\mixed"))
   "FQNs of types that should be treated as collecitons when inferring types.")
@@ -80,6 +84,12 @@ See https://wiki.php.net/rfc/static_return_type ."
   (or (phpinspect--type= type phpinspect--static-type)
       (phpinspect--type= type phpinspect--this-type)))
 
+(defsubst phpinspect--type-is-native (type)
+  (catch 'found
+    (dolist (native phpinspect-native-types)
+      (when (phpinspect--type= type native)
+        (throw 'found t)))))
+
 (cl-defmethod phpinspect--type-name ((type phpinspect--type))
   (symbol-name (phpinspect--type-name-symbol type)))
 
@@ -103,7 +113,7 @@ NAMESPACE may be nil, or a string with a namespace FQN."
          type)
 
         ;; Native type
-        ((member type phpinspect-native-types)
+        ((member type phpinspect-native-typenames)
          (concat "\\" type))
 
         ;; Relative FQN

phpinspect.el

@@ -1,4 +1,4 @@
-;;; phpinspect.el --- PHP parsing and completion package  -*- lexical-binding: t; -*-
+;; phpinspect.el --- PHP parsing and completion package  -*- lexical-binding: t; -*-
 
 ;; Copyright (C) 2021  Free Software Foundation, Inc
 
@@ -36,6 +36,7 @@
 (require 'phpinspect-util)
 (require 'phpinspect-type)
 (require 'phpinspect-index)
+(require 'phpinspect-class)
 
 (defvar-local phpinspect--buffer-index nil
   "The result of the last successfull parse + index action
@@ -51,7 +52,7 @@ phpinspect")
 (defvar phpinspect-project-root-function #'phpinspect--find-project-root
   "Function that phpinspect uses to find the root directory of a project.")
 
-(defvar phpinspect-class-filepath-function #'phpinspect-get-class-filepath
+(defvar phpinspect-type-filepath-function #'phpinspect-get-class-filepath
   "Function that phpinspect uses to find the filepath of a class by its FQN.")
 
 (defvar phpinspect-project-root-file-list
@@ -711,6 +712,8 @@ more recent"
   (eldoc-add-command 'c-electric-backspace)
 
   (phpinspect--after-save-action)
+  (phpinspect--ensure-index-thread)
+
   (add-hook 'after-save-hook #'phpinspect--after-save-action nil 'local))
 
 (defun phpinspect--after-save-action ()
@@ -1222,7 +1225,7 @@ available FQNs in a project.  This may require
 project directory before it can be used."
   (interactive (list (phpinspect--make-type
                       :name (completing-read "Class: " (phpinspect-get-all-fqns)))))
-  (find-file (phpinspect-class-filepath fqn)))
+  (find-file (phpinspect-type-filepath fqn)))
 
 (defun phpinspect-find-own-class-file (fqn)
   "`phpinspect-find-class-file', but for non-vendored classes.
@@ -1233,11 +1236,11 @@ located in \"vendor\" folder."
   (interactive (list (phpinspect--make-type
                       :name
                       (completing-read "Class: " (phpinspect-get-all-fqns "uses_own")))))
-  (find-file (phpinspect-class-filepath fqn)))
+  (find-file (phpinspect-type-filepath fqn)))
 
-(defsubst phpinspect-class-filepath (fqn)
+(defsubst phpinspect-type-filepath (fqn)
-  "Call `phpinspect-class-filepath-function' with FQN as argument."
+  "Call `phpinspect-type-filepath-function' with FQN as argument."
-  (funcall phpinspect-class-filepath-function fqn))
+  (funcall phpinspect-type-filepath-function fqn))
 
 (defun phpinspect-get-class-filepath (class &optional index-new)
   "Retrieve filepath to CLASS definition file.