Elasticsearch

.gitignore

@@ -1,9 +1,9 @@
 config.rb
 dump.rdb
 webrat.log
-*.json
+import
+coverage
 vendor/bundle
 .bundle
-Gemfile.lock
 
 .idea

Gemfile

@@ -4,15 +4,19 @@ source 'https://rubygems.org'
 gem 'unicorn'
 
 gem 'rake'
-gem 'mongo'
-gem 'bson_ext'
 gem 'sinatra'
 gem 'nokogiri', '~> 1.5.10'
 gem 'yajl-ruby'
 gem 'activesupport'
+gem 'elasticsearch'
+gem 'patron'
+gem 'hashie'
 
 group :development do
   gem 'rack-test'
+  gem 'simplecov'
   gem 'webrat'
   gem 'shotgun'
+  gem 'elasticsearch-extensions'
+  gem 'timecop'
 end

Gemfile.lock

@@ -0,0 +1,89 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.1.5)
+      i18n (~> 0.6, >= 0.6.9)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.1)
+      tzinfo (~> 1.1)
+    ansi (1.5.0)
+    docile (1.1.5)
+    elasticsearch (2.0.0)
+      elasticsearch-api (= 2.0.0)
+      elasticsearch-transport (= 2.0.0)
+    elasticsearch-api (2.0.0)
+      multi_json
+    elasticsearch-extensions (0.0.22)
+      ansi
+      ruby-prof
+    elasticsearch-transport (2.0.0)
+      faraday
+      multi_json
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
+    hashie (2.1.2)
+    i18n (0.6.11)
+    json (1.8.1)
+    kgio (2.9.2)
+    minitest (5.4.0)
+    multi_json (1.12.1)
+    multipart-post (2.0.0)
+    nokogiri (1.5.11)
+    patron (0.4.18)
+    rack (1.5.2)
+    rack-protection (1.5.3)
+      rack
+    rack-test (0.6.2)
+      rack (>= 1.0)
+    raindrops (0.13.0)
+    rake (10.3.2)
+    ruby-prof (0.16.2)
+    shotgun (0.9)
+      rack (>= 1.0)
+    simplecov (0.9.0)
+      docile (~> 1.1.0)
+      multi_json
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
+    sinatra (1.4.5)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (~> 1.3, >= 1.3.4)
+    thread_safe (0.3.4)
+    tilt (1.4.1)
+    timecop (0.7.1)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    unicorn (4.8.3)
+      kgio (~> 2.6)
+      rack
+      raindrops (~> 0.7)
+    webrat (0.7.3)
+      nokogiri (>= 1.2.0)
+      rack (>= 1.0)
+      rack-test (>= 0.5.3)
+    yajl-ruby (1.2.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport
+  elasticsearch
+  elasticsearch-extensions
+  hashie
+  nokogiri (~> 1.5.10)
+  patron
+  rack-test
+  rake
+  shotgun
+  simplecov
+  sinatra
+  timecop
+  unicorn
+  webrat
+  yajl-ruby
+
+BUNDLED WITH
+   1.10.6

Rakefile

@@ -1,5 +1,6 @@
+$LOAD_PATH.unshift '.'
 $LOAD_PATH.unshift 'lib'
-require 'tools'
+require 'utils'
 
 task :default => :test
 
@@ -12,10 +13,25 @@ end
 desc "Remove a single exception with all occurrences completely"
 task :remove_exception do
   uber_key = ENV['KEY']
-  Exceptionist::Remover.run(uber_key)
+  Utils::Remover.run(uber_key)
 end
 
-desc "Create MongoDB indexes"
-task :create_indexes do
-  Exceptionist::IndexCreator.run
+desc "Export occurrences in json"
+task :export do
+  Utils::Exporter.run
+end
+
+desc "Import occurrences from json file"
+task :import do
+  Utils::Importer.run
+end
+
+desc "Clear DB and create index with mapping"
+task :cleardb do
+  Utils::ClearDB.run
+end
+
+desc "Print mapping"
+task :mapping do
+  Utils::Mapping.run
 end

config.rb.example

@@ -11,8 +11,8 @@ require 'app'
 # ==========================
 #
 
-# Configure your MongoDB server and port
-Exceptionist.mongo = 'localhost:27017'
+# Configure your ES server and port
+Exceptionist.elasticsearch_host = 'localhost:9200'
 
 # Credentials for accessing the web app, remove if you do auth in another way
 Exceptionist.enable_authentication('username', 'password')

lib/app.rb

@@ -3,13 +3,18 @@
 require 'sinatra/base'
 require 'net/smtp'
 require 'stringio'
-require 'pp'
 
 class ExceptionistApp < Sinatra::Base
   dir = File.join(File.dirname(__FILE__), '..')
   set :views,  "#{dir}/views"
   set :public_folder, "#{dir}/public"
 
+  configure :test do
+    set :raise_errors, true
+    set :dump_errors, false
+    set :show_exceptions, false
+  end
+
   before do
     protected! if request.path_info !~ /^\/notifier_api\/v2/
   end
@@ -22,7 +27,7 @@ class ExceptionistApp < Sinatra::Base
   end
 
   get '/river' do
-    @occurrences = Occurrence.find_all
+    @occurrences = Occurrence.find
 
     @title = "River"
     erb :river
@@ -31,37 +36,49 @@ class ExceptionistApp < Sinatra::Base
   get '/projects/:project' do
     @projects = Project.all
     @current_project = Project.new(params[:project])
+
     @start = params[:start] ? params[:start].to_i : 0
+    @category = { category: params[:category] } if params[:category]
+    @terms =[{ project_name: @current_project.name }] << @category
+    @exceptions_count = UberException.count(terms: @terms)
+
     if params[:sort_by] && params[:sort_by] == 'frequent'
-      @uber_exceptions = @current_project.most_frequest_exceptions(@start)
+      @uber_exceptions = UberException.find_sorted_by_occurrences_count(terms: @terms, from: @start)
     else
-      @uber_exceptions = @current_project.latest_exceptions(@start)
+      @uber_exceptions = UberException.find(terms: @terms, from: @start)
     end
-
     @title = "Latest Exceptions for #{@current_project.name}"
     erb :index
   end
 
-  get '/projects/:project/river' do
+  get '/projects/:project/since_last_deploy' do
+    @projects = Project.all
     @current_project = Project.new(params[:project])
-    @occurrences = Occurrence.find_all(@current_project.name)
 
-    @title = "Latest Occurrences for #{@current_project.name}"
-    erb :river
-  end
+    @deploy = @current_project.last_deploy
+    raise ArgumentError, "There is no deploy for project #{@current_project.name}" unless @deploy
 
-  get '/projects/:project/new_on/:day' do
-    @day = Time.parse(params[:day])
-    @current_project = Project.new(params[:project])
-    @uber_exceptions = @current_project.new_exceptions_on(@day)
+    @category = { category: params[:category] } if params[:category]
+    @terms =[{ project_name: @current_project.name }] << @category
+    @exceptions_count = UberException.count(terms: @terms, filters: [ { range: { 'last_occurrence.occurred_at' => { gte: Helper.es_time(@deploy.occurred_at) } } } ])
 
-    message_body = erb(:new_exceptions, :layout => false)
-
-    if params[:mail_to]
-      Mailer.deliver_new_exceptions(@current_project, @day, params[:mail_to], message_body)
+    @start = params[:start] ? params[:start].to_i : 0
+    if params[:sort_by] && params[:sort_by] == 'frequent'
+      @uber_exceptions = UberException.find_since_last_deploy_ordered_by_occurrences_count(project: @current_project.name, category: params[:category], from: @start)
+    else
+      @category = { category: params[:category] }  if params[:category]
+      @uber_exceptions = UberException.find_since_last_deploy(project: @current_project.name, terms: [@category], from: @start)
     end
+    @title = "Exceptions since last deploy (#{format_time(@deploy.occurred_at)}) for project #{@current_project.name}"
+    erb :index
+  end
 
-    message_body
+  get '/projects/:project/river' do
+    @current_project = Project.new(params[:project])
+    @occurrences = Occurrence.find( filters: { term: { project_name: @current_project.name } } )
+
+    @title = "Latest Occurrences for #{@current_project.name}"
+    erb :river
   end
 
   post '/projects/:project/forget_exceptions' do
@@ -73,15 +90,10 @@ class ExceptionistApp < Sinatra::Base
 
   get '/exceptions/:id' do
     @projects = Project.all
-    @uber_exception = UberException.find(params[:id])
+    @uber_exception = UberException.get(params[:id])
     @occurrence_position = @uber_exception.occurrences_count
     @occurrence = @uber_exception.current_occurrence(@occurrence_position)
 
-    if @occurrence.nil?
-      @uber_exception.update_occurrence_count
-      redirect request.url
-    end
-
     @current_project = @occurrence.project
     @backlink = true
 
@@ -90,24 +102,42 @@ class ExceptionistApp < Sinatra::Base
   end
 
   get '/exceptions/:id/occurrences/:occurrence_position' do
-    @uber_exception = UberException.find(params[:id])
+    @uber_exception = UberException.get(params[:id])
     @occurrence_position = params[:occurrence_position].to_i
     @occurrence = @uber_exception.current_occurrence(@occurrence_position)
 
     erb :_occurrence, :layout => false
   end
 
   post '/exceptions/:id/close' do
-    @uber_exceptions = UberException.find(params[:id])
+    @uber_exceptions = UberException.get(params[:id])
     @uber_exceptions.close!
+    Exceptionist.esclient.refresh
 
     redirect to("/projects/#{@uber_exceptions.project_name}?#{Rack::Utils.unescape(params[:backparams])}")
   end
 
+  post '/exceptions/category' do
+    @uber_exceptions = UberException.get(params[:id])
+    @uber_exceptions.update( category: params[:category] )
+    Exceptionist.esclient.refresh
+
+    halt 200, 'category changed'
+  end
+
+  get '/deploys/:project' do
+    @deploys = Deploy.find_by_project(params[:project])
+    @current_project = Project.new(params[:project])
+
+    @title = "All deploys for Project #{@current_project.name}"
+
+    erb :deploy
+  end
+
   post '/notifier_api/v2/notices/?' do
     occurrence = Occurrence.from_xml(params[:data] || request.body.read)
-
     project = Project.find_by_key(occurrence.api_key)
+
     if project
       occurrence.project_name = project.name
       occurrence.save
@@ -120,6 +150,20 @@ class ExceptionistApp < Sinatra::Base
     end
   end
 
+  post '/notifier_api/v2/deploy/?' do
+    deploy = Deploy.from_json(params[:data] || request.body.read)
+    project = Project.find_by_key(deploy.api_key)
+    if project
+      deploy.project_name = project.name
+      deploy.save
+
+      "<notice><id>#{deploy.id}</id></notice>"
+    else
+      status 401
+      'Invalid API Key'
+    end
+  end
+
   helpers do
     include Rack::Utils
 

lib/boot.rb

@@ -3,13 +3,20 @@
 require 'digest'
 require 'active_support/ordered_hash'
 
-require 'mongo'
+require 'elasticsearch'
 require 'yajl'
 require 'nokogiri'
+require 'yaml'
+require 'json'
+require 'pp'
 
 require 'models/project'
 require 'models/uber_exception'
 require 'models/occurrence'
+require 'models/deploy'
 require 'models/mailer'
 
+require 'mapping/mapping_helper'
+
 require 'exceptionist'
+require 'helper'