What does a normal workflow look like?

[blankenshipz]

I'm using the nvim integration with CodeCompanion and I'm just curious what the normal workflow should be for ensuring that the embedding space is kept up to date?

For instance do I need to drop the collection and reindex when switching between commits to ensure its completely up to date?

Is there a clever way to do this automagically? Maybe something like using the git log to look at file changes between commits and push those into the vector store

Claude came up with something like this for a script to refresh the store with those semantics - note that it's keeping a "commit" store which isn't specific to a project and there's no good way to delete a file yet using the CLI so it avoided that situation (lazy llm!) maybe removing a file should be a supported cli function?

If this is a normal workflow though maybe something like it should be supported natively?

-- File: lua/plugins/vectorcode-git-sync.lua

return {
  -- This plugin depends on CodeCompanion and plenary.nvim for process handling
  dependencies = {
    "plenary.nvim",
  },
  config = function()
    local Path = require("plenary.path")
    local Job = require("plenary.job")
    local state_file = ".vectorcode-state.json"
    local max_changed_files = 100
    
    local M = {}
    
    -- Check if current directory is a git repository
    local function is_git_repo()
      local job = Job:new({
        command = "git",
        args = {"rev-parse", "--is-inside-work-tree"},
        cwd = vim.fn.getcwd(),
      })
      job:sync()
      return job.code == 0
    end
    
    -- Get current commit hash
    local function get_current_commit()
      local job = Job:new({
        command = "git",
        args = {"rev-parse", "HEAD"},
        cwd = vim.fn.getcwd(),
      })
      local result = job:sync()
      return result[1]
    end
    
    -- Read state file to get last indexed commit
    local function get_last_indexed_commit()
      local state_path = Path:new(vim.fn.getcwd() .. "/" .. state_file)
      if not state_path:exists() then
        return nil
      end
      
      local content = state_path:read()
      local last_commit = content:match('"last_indexed_commit": *"([^"]*)"')
      return last_commit
    end
    
    -- Update state file with new commit hash
    local function update_state(commit_hash)
      local state_path = Path:new(vim.fn.getcwd() .. "/" .. state_file)
      local timestamp = os.date("!%Y-%m-%dT%H:%M:%SZ")
      local content = string.format('{"last_indexed_commit": "%s", "last_update": "%s"}', 
                                   commit_hash, timestamp)
      state_path:write(content, "w")
    end
    
    -- Get list of files changed between commits
    local function get_changed_files(last_commit, current_commit)
      local job = Job:new({
        command = "git",
        args = {"diff", "--name-only", last_commit, current_commit},
        cwd = vim.fn.getcwd(),
      })
      return job:sync()
    end
    
    -- Run full vectorcode indexing
    local function full_indexing()
      vim.notify("Performing full vectorcode indexing...", vim.log.levels.INFO)
      
      -- Optional: Drop existing collection
      Job:new({
        command = "vectorcode",
        args = {"drop", Path:new(vim.fn.getcwd()):filename()},
        cwd = vim.fn.getcwd(),
      }):start()
      
      -- Run vectorcode on current directory
      local index_job = Job:new({
        command = "vectorcode",
        args = {"vectorise", "."},
        cwd = vim.fn.getcwd(),
        on_exit = function(_, code)
          if code == 0 then
            local current_commit = get_current_commit()
            update_state(current_commit)
            vim.notify("✅ Full vectorcode indexing complete", vim.log.levels.INFO)
          else
            vim.notify("❌ Vectorcode indexing failed", vim.log.levels.ERROR)
          end
        end
      })
      
      index_job:start()
    end
    
    -- Process only changed files
    local function incremental_update(files)
      vim.notify("Updating embeddings for " .. #files .. " changed files...", vim.log.levels.INFO)
      
      local current_commit = get_current_commit()
      local processed = 0
      
      for _, file in ipairs(files) do
        if file ~= "" then
          local file_path = Path:new(vim.fn.getcwd() .. "/" .. file)
          
          if file_path:exists() then
            Job:new({
              command = "vectorcode",
              args = {"vectorise", file},
              cwd = vim.fn.getcwd(),
              on_exit = function(_, code)
                processed = processed + 1
                if processed == #files then
                  update_state(current_commit)
                  vim.notify("✅ Incremental vectorcode update complete", vim.log.levels.INFO)
                end
              end
            }):start()
          else
            processed = processed + 1
            -- Note: Handling deleted files is complex
            -- You might want to implement specific handling here
          end
        end
      end
    end
    
    -- Main sync function
    function M.sync_vectorcode()
      if not is_git_repo() then
        vim.notify("Not in a git repository", vim.log.levels.ERROR)
        return
      end
      
      local current_commit = get_current_commit()
      local last_commit = get_last_indexed_commit()
      
      if not last_commit then
        full_indexing()
        return
      end
      
      local changed_files = get_changed_files(last_commit, current_commit)
      local changed_count = #changed_files
      
      if changed_count > max_changed_files then
        vim.notify("Large number of changes detected. Performing full indexing...", vim.log.levels.INFO)
        full_indexing()
      elseif changed_count > 0 then
        incremental_update(changed_files)
      else
        vim.notify("No files changed since last indexing. Nothing to do.", vim.log.levels.INFO)
      end
    end
    
    -- Create user commands
    vim.api.nvim_create_user_command("VectorcodeSync", function()
      M.sync_vectorcode()
    end, {})
    
    -- Auto-sync on events if desired
    vim.api.nvim_create_autocmd({"BufWritePost"}, {
      pattern = "*",
      callback = function()
        -- Uncomment to enable auto-sync on save
        -- vim.defer_fn(M.sync_vectorcode, 1000)
      end
    })
    
    -- Setup CodeCompanion integration
    -- Add a hook for CodeCompanion - this depends on its API
    local has_companion, companion = pcall(require, "codecompanion")
    if has_companion then
      -- This part depends on CodeCompanion's actual API
      -- Assuming it has a register_hook function or similar
      if companion.register_hook then
        companion.register_hook("before_query", function()
          -- Optionally sync before each query
          -- M.sync_vectorcode()
          return true -- continue with query
        end)
      end
    end
    
    -- Return the module
    return M
  end,
}

[Davidyz]

Hi, to refreshing embeddings you can simply re-vectorise files. This will remove existing embeddings for the files and ingest new embeddings to the database. To do this automatically, you can follow these steps in the wiki: https://github.com/Davidyz/VectorCode/wiki/Tips-and-Tricks#git-hooks or simply use the vectorcode update command. The git hook method in the wiki will track new files in dirs that you marked as included, while the update subcommand only track existing files. Both methods will detect removed files and remove corresponding embeddings.

Edit: I do plan to introduce the vectorise command to the codecompanion tool, but ATM I think migration to chroma 1.0.0 has higher priority, because it's fairly straightforward to set up an autocmd that calls vectorcode update (this will imply a risk of having 2 processes trying to vectorise at the same time tho. This is why I recommend a standalone server over the bundled chroma server).